convert_to_BIO.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "consolidated-kuwait",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import string\n",
    "import seqeval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "worldwide-pointer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "convert_gold=True\n",
    "goldfolder='multitask4mix_bi' ## folder containing gold file\n",
    "#goldfolder = 'mlee_processed_sent'\n",
    "goldfile='testmlevex.csv' \n",
    "#goldfile='test.csv'\n",
    "golddf = pd.read_csv(os.path.join(goldfolder,goldfile))\n",
    "opfile=''\n",
    "\n",
    "convert_pred=True\n",
    "predfolder='results/multitask4mixbidspec2verbose2_prin_ml_1024_50_bs50_seq3_4bs_2DAYS' ## folder containing pred file\n",
    "#predfile='EV-EX_wikievent_sent_neg_processed.csv'\n",
    "print([p for p in os.listdir(predfolder) if 'EX' in p])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "id": "alert-slovenia",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(875, 15) (875, 13)\n"
     ]
    }
   ],
   "source": [
    "predfile = 'EV-EX_mlee_processed_sent.csv' ### if seq3 is not in predfile, i.e. greedy search pred, set this to predfile\n",
    "\n",
    "\n",
    "if('seq3' in predfolder):\n",
    "    predfile = 'compare_top_3.csv'\n",
    "preddf = pd.read_csv(os.path.join(predfolder,predfile))\n",
    "print(preddf.shape,golddf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "partial-survivor",
   "metadata": {},
   "outputs": [],
   "source": [
    "textcol_g = 'question'\n",
    "labcol_g='src_label'\n",
    "labcol_p = 'pred'\n",
    "if('seq3' in predfolder):\n",
    "    labcol_p='top_1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "heavy-hepatitis",
   "metadata": {},
   "outputs": [],
   "source": [
    "if('subtask' in golddf.columns and golddf.subtask.nunique()>1):\n",
    "    golddf = golddf[golddf.subtask=='EV-EX']\n",
    "    \n",
    "    if(len(preddf)>len(golddf)):\n",
    "        preddf = preddf.iloc[golddf.index]\n",
    "    \n",
    "    #golddf = golddf.reset_index(drop=True)\n",
    "    #preddf = preddf.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "id": "protecting-hypothetical",
   "metadata": {},
   "outputs": [],
   "source": [
    "golddf = golddf.reset_index(drop=True)\n",
    "preddf = preddf.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "id": "grave-hungary",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['text', 'gold', 'top_1', 'top_2', 'top_3', 'top_2_combined', 'cor1',\n",
       "       'cor2', 'cor3', 'option_1', 'cor_option_1', 'option_2', 'cor_option_2',\n",
       "       'option_3', 'cor_option_3'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preddf.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "collaborative-criterion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'EV-EX</s>mlee_processed_sent</s>Adenovirus-mediated gene transfer of endostatin in vivo results in high level of transgene expression and inhibition of tumor growth and metastases.'"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "golddf['text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "usual-cause",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((875, 13), (875, 15))"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "golddf.shape,preddf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "designing-boxing",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "id": "nervous-methodology",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_BIO(texts,labels,verbose=False):\n",
    "    \n",
    "    multi=0\n",
    "    mlt=[]\n",
    "    gold=[]\n",
    "    glabs=[]\n",
    "    \n",
    "    textcol_g = 'text'\n",
    "    labcol_g = 'src_label'\n",
    "    dftemp = pd.DataFrame(data={'text':texts,'src_label':labels})\n",
    "\n",
    "    for ix,r in dftemp.iterrows():\n",
    "        \n",
    "        mult=False\n",
    "        \n",
    "        glabs=[]\n",
    "        texto = r[textcol_g].split('</s>')[-1].translate(str.maketrans('', '', string.punctuation))\n",
    "        \n",
    "        \n",
    "        \n",
    "        events = [p.strip() for p in str(r[labcol_g]).split('|')]\n",
    "        \n",
    "        labs = [p.split('->')[0].strip() for p in events]\n",
    "        \n",
    "        if(verbose):\n",
    "            print(r['src_label'])\n",
    "            print(texto)\n",
    "            print('***************')\n",
    "            \n",
    "        senlen = len(texto.split())\n",
    "        #print(texto,senlen)\n",
    "\n",
    "\n",
    "        if('NONE' in labs or 'nan' in labs):\n",
    "            #print('NONE')\n",
    "            glabs = ['O']*senlen\n",
    "            gold.append(glabs)\n",
    "            mlt.append(False)\n",
    "            #print(glabs,len(glabs))\n",
    "            continue\n",
    "\n",
    "        events = [str(p) for p in events if '->' in str(p)]\n",
    "        \n",
    "        labs = [p.split('->')[0].strip() for p in events]\n",
    "        clfs = [p.split('->')[1].strip() for p in events]\n",
    "        dictword={}\n",
    "        mct=0\n",
    "        ml=False\n",
    "        for ind,g in enumerate(labs):\n",
    "            startpos = texto.find(g)\n",
    "            if(labs.count(g)>1):\n",
    "                mct+=1\n",
    "                \n",
    "            if(len(g.split())>1):\n",
    "                ml=True\n",
    "                if('ml' in predfolder or 'ML' in predfolder):\n",
    "                    ml=False \n",
    "                    continue\n",
    "            \n",
    "            for i,word in enumerate(g.split()):\n",
    "                \n",
    "                \n",
    "                \n",
    "                if(i==0):\n",
    "                    #print('found')\n",
    "                    #startpos = texto.find(word,startpos)\n",
    "                    \n",
    "                    \n",
    "                    dictword[word+str(startpos)] = 'B-'+clfs[ind]\n",
    "                else:\n",
    "                    \n",
    "                    mult=True\n",
    "                    #print('*******')\n",
    "                    pos = texto.find(word,startpos+1)\n",
    "                    if(word_clf):\n",
    "                        dictword[word+str(startpos)] = 'B-'+clfs[ind]\n",
    "                    else:\n",
    "                        dictword[word+str(pos)] = 'I-'+clfs[ind]\n",
    "            multi+=mult\n",
    "            \n",
    "            mult=False\n",
    "        pos=-1\n",
    "        mlt.append(ml)\n",
    "        #print(dictword)\n",
    "        for word in texto.split():\n",
    "            pos = texto.find(word,pos+1)\n",
    "            if(word+str(pos) in dictword.keys()):\n",
    "                glabs.append(dictword[word+str(pos)])\n",
    "            else:\n",
    "                glabs.append('O')\n",
    "\n",
    "        if(verbose):\n",
    "            print(glabs,len(glabs))\n",
    "        gold.append(glabs)\n",
    "        #print('------------------')\n",
    "        #break\n",
    "\n",
    "    print('multi-word triggers ',multi)\n",
    "    print('multi-class triggers ',mct)\n",
    "    return gold,mlt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "id": "hydraulic-helmet",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_none(gold,pred):\n",
    "    g=[]\n",
    "    p=[]\n",
    "    for l1,l2 in zip(gold,pred):\n",
    "        ge = [elem for elem in l1 if elem!='O']\n",
    "        pe = [l2[ix] for ix,elem in enumerate(l1) if elem!='O']\n",
    "        g.append(ge)\n",
    "        p.append(pe)\n",
    "    return g,p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "piano-australian",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "id": "minus-elizabeth",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_clf=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "id": "eligible-contributor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'associated->regulation | metastasis->localization | angiogenesis->blood_vessel_development | expression->gene_expression | coexpression->gene_expression | role->regulation'"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preddf[labcol_p][36]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "id": "african-cuisine",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi-word triggers  0\n",
      "multi-class triggers  0\n",
      "multi-word triggers  0\n",
      "multi-class triggers  0\n",
      "875 875\n"
     ]
    }
   ],
   "source": [
    "goldlabels,mlt1 = get_BIO(golddf[textcol_g],golddf[labcol_g])\n",
    "predlabels,mlt2 = get_BIO(golddf[textcol_g],preddf[labcol_p])\n",
    "print(len(goldlabels),len(predlabels))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prime-vault",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "id": "critical-eleven",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"df1 = pd.DataFrame(data={'pred':preddf[labcol_p],'gold':golddf[labcol_g],'pred_multi':mlt2})\\nprint(df1[df1.pred_multi==True]['pred'].iloc[1])\\nprint(df1[df1.pred_multi==True]['gold'].iloc[1])\""
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''df1 = pd.DataFrame(data={'pred':preddf[labcol_p],'gold':golddf[labcol_g],'pred_multi':mlt2})\n",
    "print(df1[df1.pred_multi==True]['pred'].iloc[1])\n",
    "print(df1[df1.pred_multi==True]['gold'].iloc[1])'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "id": "extraordinary-equilibrium",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "875 875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-370-d2b6291ac4b6>:5: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  predlabels = np.asarray(predlabels)\n",
      "<ipython-input-370-d2b6291ac4b6>:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  goldlabels = np.asarray(goldlabels)\n"
     ]
    }
   ],
   "source": [
    "goldpos,predpos = remove_none(goldlabels,predlabels)\n",
    "print(len(goldpos),len(predpos))\n",
    "\n",
    "\n",
    "predlabels = np.asarray(predlabels)\n",
    "goldlabels = np.asarray(goldlabels)\n",
    "\n",
    "if('NONE' in golddf.src_label.unique()):\n",
    "    indices = golddf[~golddf.src_label.str.contains('NONE')].index\n",
    "\n",
    "    predpos2 = predlabels[indices]\n",
    "    goldpos2 = goldlabels[indices]\n",
    "else:\n",
    "    predpos2=predlabels\n",
    "    goldpos2=goldlabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "id": "linear-mayor",
   "metadata": {},
   "outputs": [],
   "source": [
    "sr = pd.Series(mlt2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unable-spare",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "id": "demonstrated-plane",
   "metadata": {},
   "outputs": [],
   "source": [
    "#goldlabels = goldlabels[2:3]\n",
    "#predlabels = predlabels[2:3]\n",
    "#goldpos,predpos = remove_none(goldlabels,predlabels)\n",
    "#print(len(goldpos),len(predpos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "id": "attempted-tomorrow",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(875, 703)"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(goldlabels),len(goldpos2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "id": "foster-throw",
   "metadata": {},
   "outputs": [],
   "source": [
    "from seqeval.metrics import f1_score,accuracy_score,precision_score,recall_score\n",
    "\n",
    "evals=['normal','POS tokens','POS sentences']\n",
    "precs = [precision_score(goldlabels,predlabels),precision_score(goldpos,predpos),precision_score(goldpos2,predpos2)]\n",
    "recs = [recall_score(goldlabels,predlabels),recall_score(goldpos,predpos),recall_score(goldpos2,predpos2)]\n",
    "f1s = [f1_score(goldlabels,predlabels),f1_score(goldpos,predpos),f1_score(goldpos2,predpos2)]\n",
    "accs = [accuracy_score(goldlabels,predlabels),accuracy_score(goldpos,predpos),accuracy_score(goldpos2,predpos2)]\n",
    "\n",
    "metrics = pd.DataFrame(data={'evaltype':evals,'precision':precs,'recall':recs,'f1-score':f1s,'accuracy':accs})\n",
    "\n",
    "#print(f1_score(goldlabels,predlabels),accuracy_score(goldlabels,predlabels))\n",
    "#print(f1_score(goldpos,predpos),accuracy_score(goldpos,predpos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "whole-underground",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>evaltype</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>normal</td>\n",
       "      <td>0.759661</td>\n",
       "      <td>0.789763</td>\n",
       "      <td>0.774420</td>\n",
       "      <td>0.961595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>POS sentences</td>\n",
       "      <td>0.783725</td>\n",
       "      <td>0.789763</td>\n",
       "      <td>0.786732</td>\n",
       "      <td>0.958017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        evaltype  precision    recall  f1-score  accuracy\n",
       "0         normal   0.759661  0.789763  0.774420  0.961595\n",
       "2  POS sentences   0.783725  0.789763  0.786732  0.958017"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics[metrics.evaltype.isin(['normal','POS sentences'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "convertible-kitty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'results/multitask4mixbidspec2verbose2_prin_ml_1024_50_bs50_seq3_4bs_2DAYS'"
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predfolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "id": "premier-vertical",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'odf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-377-2d38be2641d7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'maven'\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpredfolder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0modf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'odf' is not defined"
     ]
    }
   ],
   "source": [
    "if('maven' not in predfolder):\n",
    "    odf.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cardiac-peripheral",
   "metadata": {},
   "outputs": [],
   "source": [
    "from seqeval.metrics import classification_report\n",
    "print(classification_report(goldlabels,predlabels,digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "electrical-isolation",
   "metadata": {},
   "outputs": [],
   "source": [
    "predfolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dried-forestry",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "important-soviet",
   "metadata": {},
   "outputs": [],
   "source": [
    "#list(goldlabels[0]),list(predlabels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "peripheral-following",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(golddf[textcol_g][0].split())\n",
    "#goldlabels[0]\n",
    "golddf[textcol_g][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "widespread-county",
   "metadata": {},
   "outputs": [],
   "source": [
    "gl=[]\n",
    "pl=[]\n",
    "st=[]\n",
    "for ix,p in enumerate(goldlabels):\n",
    "    #print(p)\n",
    "    \n",
    "    gl+=p\n",
    "    pl+=predlabels[ix]\n",
    "    #break\n",
    "#(goldlabels[0])\n",
    "\n",
    "st = [s for s in gl if s!='O']\n",
    "print(pd.Series(st).nunique())\n",
    "print(len(gl),len(pl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "subtle-reading",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics as skmt\n",
    "\n",
    "norm=True\n",
    "if(norm):\n",
    "    print('incl NONE class')\n",
    "    print('precison', skmt.precision_score(gl,pl,average='micro'))\n",
    "    print('recall', skmt.recall_score(gl,pl,average='micro'))\n",
    "    print('micro', skmt.f1_score(gl,pl,average='micro'))\n",
    "    print('macro', skmt.f1_score(gl,pl,average='macro'))\n",
    "    print('weighted', skmt.f1_score(gl,pl,average='weighted'))\n",
    "    print('-------------------')\n",
    "    print('not incl NONE class')\n",
    "    print('precison', skmt.precision_score(gl,pl,average='micro',labels=st))\n",
    "    print('recall', skmt.recall_score(gl,pl,average='micro',labels=st))\n",
    "    print('micro', skmt.f1_score(gl,pl,average='micro',labels=st))\n",
    "    print('macro', skmt.f1_score(gl,pl,average='macro',labels=st))\n",
    "    print('weighted', skmt.f1_score(gl,pl,average='weighted',labels=st))\n",
    "else:\n",
    "    gl = [gl]\n",
    "    pl = [pl]\n",
    "    print('incl NONE class')\n",
    "    print('precison', skmt.precision_score(gl,pl,average='micro'))\n",
    "    print('recall', skmt.recall_score(gl,pl,average='micro'))\n",
    "    print('micro', skmt.f1_score(gl,pl,average='micro'))\n",
    "    print('macro', skmt.f1_score(gl,pl,average='macro'))\n",
    "    print('weighted', skmt.f1_score(gl,pl,average='weighted'))\n",
    "    print('-------------------')\n",
    "    print('not incl NONE class')\n",
    "    print('precison', skmt.precision_score(gl,pl,average='micro',labels=st))\n",
    "    print('recall', skmt.recall_score(gl,pl,average='micro',labels=st))\n",
    "    print('micro', skmt.f1_score(gl,pl,average='micro',labels=st))\n",
    "    print('macro', skmt.f1_score(gl,pl,average='macro',labels=st))\n",
    "    print('weighted', skmt.f1_score(gl,pl,average='weighted',labels=st))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "living-video",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "viral-substitute",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "peaceful-gathering",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "essential-watts",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''for i in range(5):\n",
    "    print(golddf[labcol_g][i])\n",
    "    print(preddf[labcol_p][i])\n",
    "    print(goldlabels[i])\n",
    "    print(predlabels[i])'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "inappropriate-needle",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "skilled-rolling",
   "metadata": {},
   "outputs": [],
   "source": [
    "#len(golddf[golddf.src_label=='NONE'])/len(golddf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "western-daisy",
   "metadata": {},
   "outputs": [],
   "source": [
    "#2659/3456"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alien-worse",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}