[THIS] multitask multidata scoring entity sep type.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "tender-thinking",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import jsonlines\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "forty-quarterly",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "use_sent_mask=False\n",
    "if(use_sent_mask):\n",
    "    maskfolder='sent_classifier_wikievent_512_60_1bs_2DAYS'\n",
    "    mask=open('results/'+maskfolder+'/generated_predictions.txt').readlines()\n",
    "\n",
    "    \n",
    "using_binary_clf_top = False\n",
    "\n",
    "predfolder = 'multitask4mixbidspec2verbose2_prin_ml_1024_50_4bs_2DAYS'\n",
    "goldfolder = 'multitask4mix_bi'\n",
    "\n",
    "\n",
    "#lab='src21'\n",
    "lab=''\n",
    "relcol='src_label'\n",
    "\n",
    "golddf = pd.read_csv(goldfolder+'/testml.csv')\n",
    "#golddf = pd.read_csv(goldfolder+'/test.csv')\n",
    "#clfs = pd.read_csv(goldfolder+'/testcasieevclf.csv')\n",
    "\n",
    "\n",
    "ds = ''\n",
    "gold = golddf[relcol]\n",
    "#txt = golddf['text']\n",
    "#gold=gtype\n",
    "\n",
    "if('FIXED' not in predfolder):\n",
    "    \n",
    "    #if('tkinstr' in predfolder or 't5L' in predfolder):\n",
    "    #    pred = open('results/'+predfolder+'/test_generations.txt').readlines()\n",
    "    #    pred = [p[0:-1] for p in pred] # removing \\n suffix\n",
    "        \n",
    "    #else:\n",
    "    pred = open('results/'+predfolder+'/generated_predictions.txt').readlines()\n",
    "    pred[-1] = pred[-1]+'\\n'\n",
    "    pred = [p[2:-2] for p in pred] # removing b' prefix and \\n suffix\n",
    "\n",
    "if(using_binary_clf_top):\n",
    "    pred = open('results/'+predfolder+'/generated_predictions_new.txt').readlines()\n",
    "    print('fixed')\n",
    "pred = pd.Series(pred)\n",
    "labelsep = ' | ' # separates multiple entity+type combos\n",
    "\n",
    "termsep = '->' # separates entity and type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "native-mission",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2625 2625\n"
     ]
    }
   ],
   "source": [
    "print(len(gold),len(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "perfect-strand",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pred[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "plastic-storm",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pred.iloc[-1]+'e'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "regional-portsmouth",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"rem = pred.iloc[-1]+'e'\\npred = list(pred[:-1])\\npred.append('contact.threatencoerce')\\npred.append(rem+'e')\\n#+['contact.threatencoerce']+[rem+'e']\""
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''rem = pred.iloc[-1]+'e'\n",
    "pred = list(pred[:-1])\n",
    "pred.append('contact.threatencoerce')\n",
    "pred.append(rem+'e')\n",
    "#+['contact.threatencoerce']+[rem+'e']'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "violent-review",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       planned_process | gene_expression | positive_r...\n",
       "1       metastases | results | inhibition | mediated |...\n",
       "2       inhibition->negative_regulation | transfer->pl...\n",
       "3       planned_process | blood_vessel_development | n...\n",
       "4                     Inhibition | therapy | angiogenesis\n",
       "                              ...                        \n",
       "2620                                   neovascularization\n",
       "2621         neovascularization->blood_vessel_development\n",
       "2622                                      planned_process\n",
       "2623                                       prefabrication\n",
       "2624                      prefabrication->planned_process\n",
       "Name: src_label, Length: 2625, dtype: object"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "competitive-corps",
   "metadata": {},
   "outputs": [],
   "source": [
    "if(len(pred)>len(gold)):\n",
    "    pred = [p for p in pred if len(p)>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "flush-union",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2625, 2625)"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gold),len(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "amateur-replication",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('The event triggers in this text are mediated, inhibition, growth, metastases, transfer, expression',\n",
       " 'The event triggers and their types in this text are: growth is a trigger of type growth, inhibition is a trigger of type negative_regulation, expression is a trigger of type gene_expression, transfer is a trigger of type planned_process, results is a trigger of type positive_regulation, mediated is a trigger of type positive_regulation, metastases is a trigger of type localization',\n",
       " 'The types of events in this text are negative_regulation, blood_vessel_development, planned_process')"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred[1],pred[2],pred[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "passing-compact",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "2625\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if(use_sent_mask):\n",
    "    \n",
    "    pred_o = pred.copy()\n",
    "    pred=[]\n",
    "    for i,r in enumerate(pred_o):\n",
    "        if(mask[i][2:-2]=='False'):\n",
    "            #print('Found')\n",
    "            pred.append('NONE')\n",
    "        else:\n",
    "            pred.append(r)\n",
    "    ### the ones which are 0, conver preds to NONE\n",
    "\n",
    "print(len([p for p in pred if p=='NONE']))\n",
    "print(len(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "above-magnet",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "beautiful-wilderness",
   "metadata": {},
   "outputs": [],
   "source": [
    "predz=[]\n",
    "sb = golddf['subtask']\n",
    "\n",
    "if('src21' in predfolder or lab=='src21'):\n",
    "    ## reversed outputs/ convert to src_label1 format\n",
    "    \n",
    "    for p in pred:\n",
    "        fin=[]\n",
    "        events = [x.strip() for x in p.split('|')]\n",
    "        for e in events:\n",
    "            #print(e)\n",
    "            if('->' not in e):\n",
    "                typ,trigs=['','']\n",
    "            else:\n",
    "                typ,trigs=e.split('->')[0:2]\n",
    "            \n",
    "            for tg in trigs.split('@'):\n",
    "                fin.append(tg+'->'+typ)\n",
    "        if(p.strip()=='->'):\n",
    "            predz.append('NONE')\n",
    "        else:\n",
    "            predz.append(' | '.join(fin))\n",
    "        \n",
    "    pred = predz.copy()\n",
    "\n",
    "    \n",
    "if('src12' in predfolder or lab=='src12'):\n",
    "    \n",
    "    for idx,p in enumerate(pred):\n",
    "        nm=[]\n",
    "        if('NONE' in p):\n",
    "            predz.append('NONE')\n",
    "            continue\n",
    "        \n",
    "        if('EV-EX' not in sb[idx]):\n",
    "            predz.append(p)\n",
    "            continue\n",
    "        events = [x.strip() for x in p.split('|')]\n",
    "        #print(events,sb.iloc[idx])\n",
    "        for e in events:\n",
    "            if('->' not in e):\n",
    "                continue\n",
    "            tg = e.split('->')[0]\n",
    "            tps = e.split('->')[1].split('@')\n",
    "            nm += [tg+'->'+p for p in tps]\n",
    "        if(len(nm)==0):\n",
    "            nm = ['NONE']\n",
    "        predz.append(' | '.join(nm))\n",
    "        \n",
    "    pred = predz.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "conventional-likelihood",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "planned_process | gene_expression | positive_regulation | localization | negative_regulation | growth\n",
      "\n",
      "The types of events in this text are negative_regulation, localization, gene_expression, growth, positive_regulation, planned_process\n",
      "**************************\n",
      "metastases | results | inhibition | mediated | transfer | expression | growth\n",
      "\n",
      "The event triggers in this text are mediated, inhibition, growth, metastases, transfer, expression\n",
      "**************************\n",
      "inhibition->negative_regulation | transfer->planned_process | mediated->positive_regulation | expression->gene_expression | metastases->localization | growth->growth | results->positive_regulation\n",
      "\n",
      "The event triggers and their types in this text are: growth is a trigger of type growth, inhibition is a trigger of type negative_regulation, expression is a trigger of type gene_expression, transfer is a trigger of type planned_process, results is a trigger of type positive_regulation, mediated is a trigger of type positive_regulation, metastases is a trigger of type localization\n"
     ]
    }
   ],
   "source": [
    "print(gold[0]+'\\n\\n'+pred[0])\n",
    "print('**************************')\n",
    "print(gold[1]+'\\n\\n'+pred[1])\n",
    "print('**************************')\n",
    "print(gold[2]+'\\n\\n'+pred[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "statistical-intervention",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if('verb' in predfolder):\n",
    "    p1 = pred.str.replace('The types of events in this text are ','').str.replace('The event triggers in this text are ','').str.replace('Events in this text: ','').str.replace('The event triggers and their types in this text are: ','')\n",
    "    p2 = p1.str.replace(' is a trigger of type ','->').str.replace(',',' |')\n",
    "    pred = p2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hundred-worker",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "pleased-japan",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "planned_process | gene_expression | positive_regulation | localization | negative_regulation | growth\n",
      "\n",
      "negative_regulation | localization | gene_expression | growth | positive_regulation | planned_process\n",
      "**************************\n",
      "metastases | results | inhibition | mediated | transfer | expression | growth\n",
      "\n",
      "mediated | inhibition | growth | metastases | transfer | expression\n",
      "**************************\n",
      "inhibition->negative_regulation | transfer->planned_process | mediated->positive_regulation | expression->gene_expression | metastases->localization | growth->growth | results->positive_regulation\n",
      "\n",
      "growth->growth | inhibition->negative_regulation | expression->gene_expression | transfer->planned_process | results->positive_regulation | mediated->positive_regulation | metastases->localization\n"
     ]
    }
   ],
   "source": [
    "print(gold[0]+'\\n\\n'+pred[0])\n",
    "print('**************************')\n",
    "print(gold[1]+'\\n\\n'+pred[1])\n",
    "print('**************************')\n",
    "print(gold[2]+'\\n\\n'+pred[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "productive-louis",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "respected-basketball",
   "metadata": {},
   "outputs": [],
   "source": [
    "def div(a,b):\n",
    "    if b==0:\n",
    "        return 0;\n",
    "    else:\n",
    "        return a/b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "bearing-question",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"def macro(gold,pred,task,dataset):\\n    classes=[]\\n    f1s=[]\\n    for i,r in clfs.iterrows():\\n        c = [p.strip() for p in r[relcol].split('|')]\\n        classes+=c\\n        \\n    print(pd.Series(classes).nunique(), ' unique classes')\\n    cdt = {key: [] for key in classes}\\n    \\n    for cf in classes:\\n        print('cf ',cf)\\n        indxs = clfs[clfs[relcol].str.contains(cf)].index\\n                \\n    \\n    print('avged f1 across classes: ',np.sum(f1s)/len(f1s))\\n    ### get clf list from ev-clf dataset.split on '|' and strip\\n    ### get values by adding lists and getting unique\\n    ### for each class, get indices (will be same for evex and evclf? added in same order right).\\n    ##### dict. classname:[indices]\\n    ### for those sentences run scoring()\\n    ### average over all classes.\""
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''def macro(gold,pred,task,dataset):\n",
    "    classes=[]\n",
    "    f1s=[]\n",
    "    for i,r in clfs.iterrows():\n",
    "        c = [p.strip() for p in r[relcol].split('|')]\n",
    "        classes+=c\n",
    "        \n",
    "    print(pd.Series(classes).nunique(), ' unique classes')\n",
    "    cdt = {key: [] for key in classes}\n",
    "    \n",
    "    for cf in classes:\n",
    "        print('cf ',cf)\n",
    "        indxs = clfs[clfs[relcol].str.contains(cf)].index\n",
    "                \n",
    "    \n",
    "    print('avged f1 across classes: ',np.sum(f1s)/len(f1s))\n",
    "    ### get clf list from ev-clf dataset.split on '|' and strip\n",
    "    ### get values by adding lists and getting unique\n",
    "    ### for each class, get indices (will be same for evex and evclf? added in same order right).\n",
    "    ##### dict. classname:[indices]\n",
    "    ### for those sentences run scoring()\n",
    "    ### average over all classes.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "patent-costume",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scoring(gold,pred,task,indices=None,verbose=True,skipneg=True):\n",
    "    tp = 0\n",
    "    predn=0\n",
    "    goldn=0\n",
    "    skipped=0\n",
    "    \n",
    "    x=0\n",
    "    cor=[]\n",
    "    posind=[]\n",
    "\n",
    "    for ind,row in enumerate(pred):\n",
    "        #row = row.replace(': ','->')\n",
    "        \n",
    "        if(row=='->'):\n",
    "            row='NONE'\n",
    "        if(indices is not None):\n",
    "            gold2 = gold[indices[x]]\n",
    "        x+=1\n",
    "        \n",
    "        if(str(gold2)=='nan'):\n",
    "            gold2=''\n",
    "        \n",
    "        '''\n",
    "        if(gold2=='NONE'):\n",
    "            continue\n",
    "        '''\n",
    "        \n",
    "        posind.append(ind)    \n",
    "        row = row.split('|')\n",
    "        gold2 = gold2.split('|')\n",
    "        \n",
    "        \n",
    "        gold2 = [g.strip() for g in gold2]\n",
    "        row = [g.strip() for g in row]\n",
    "        \n",
    "        r2 = row.copy()\n",
    "        g2 = gold2.copy()\n",
    "        \n",
    "        #print(gold2,'\\n\\n',row)\n",
    "        #break\n",
    "        r2 = [r for r in row if len(r)>1]\n",
    "        \n",
    "        \n",
    "        #gold[i] = str(gold[i]).split(' |')\n",
    "        \n",
    "        if(task=='EV-CLF'):\n",
    "            row = list(set(row))\n",
    "            gold2 = list(set(gold2))\n",
    "            \n",
    "        row.sort()\n",
    "        gold2.sort()\n",
    "        cor.append(row==gold2)\n",
    "        \n",
    "        if(skipneg and gold2[0]=='NONE'):\n",
    "            skipped+=1\n",
    "            continue\n",
    "\n",
    "        \n",
    "        for elem in row:\n",
    "            if(len(elem)<1):\n",
    "                continue\n",
    "            \n",
    "                \n",
    "            \n",
    "            if(elem in gold2): # exact phrase present\n",
    "                tp+=1\n",
    "\n",
    "            \n",
    "                \n",
    "                # NEW\n",
    "                gold2.remove(elem)\n",
    "            \n",
    "        gold2 = g2.copy()\n",
    "        for g in gold2: # part of pred phrase present in gold\n",
    "            if(len(g)<1):\n",
    "                continue\n",
    "            \n",
    "        \n",
    "        predn+=len(row)-len([p for p in row if len(p)<1])\n",
    "        goldn+=len(gold2)-len([p for p in gold2 if len(p)<1])\n",
    "        #print('tp: ',tp,' predn: ',predn,' goldn: ',goldn)\n",
    "        #print('tp_rel: ',tp_relaxed)\n",
    "        #print('tp_rel2: ',tp_relaxed2)\n",
    "\n",
    "    p = div(tp,predn)\n",
    "    r = div(tp,goldn)\n",
    "    f1 = div(2*p*r,(p+r))\n",
    "\n",
    "    if(verbose):\n",
    "        print('skipped: ',skipped)\n",
    "        print('\\nPrecision: ',p,'\\nRecall: ',r,'\\nF1-score: ',f1)\n",
    "\n",
    "    \n",
    "    '''\n",
    "    prel = div(tp_relaxed,predn)\n",
    "    rrel = div(tp_relaxed,goldn)\n",
    "    f1rel = div(2*prel*rrel,(prel+rrel))\n",
    "    \n",
    "    print('\\nRELAXED SCORING 1\\nPrecision: ',prel,'\\nRecall: ',rrel,'\\nF1-score: ',f1rel)\n",
    "\n",
    "    prel2 = div(tp_relaxed2,predn)\n",
    "    rrel2 = div(tp_relaxed2,goldn)\n",
    "    f1rel2 = div(2*prel2*rrel2,(prel2+rrel2))\n",
    "\n",
    "    print('\\nRELAXED SCORING 2\\nPrecision: ',prel2,'\\nRecall: ',rrel2,'\\nF1-score: ',f1rel2)\n",
    "    '''\n",
    "    return posind,cor,p,r,f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mexican-jumping",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "dietary-revelation",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "dataset:  mlee_processed_sent\n",
      "------------------------------------------\n",
      "subtask:  EV-CLF\n",
      "875 875\n",
      "skipped:  172\n",
      "\n",
      "Precision:  0.8223684210526315 \n",
      "Recall:  0.8121677495569994 \n",
      "F1-score:  0.8172362555720654\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.8096008629989212 \n",
      "Recall:  0.8048257372654155 \n",
      "F1-score:  0.8072062382360848\n",
      "875 875 875 875 875\n",
      "------------------------------------------\n",
      "subtask:  EV-ID\n",
      "875 875\n",
      "skipped:  172\n",
      "\n",
      "Precision:  0.7912957467853611 \n",
      "Recall:  0.7916872835230084 \n",
      "F1-score:  0.7914914667326243\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.78125 \n",
      "Recall:  0.786593707250342 \n",
      "F1-score:  0.7839127471029311\n",
      "875 875 875 875 875\n",
      "------------------------------------------\n",
      "subtask:  EV-EX\n",
      "875 875\n",
      "skipped:  172\n",
      "\n",
      "Precision:  0.7417315175097277 \n",
      "Recall:  0.7545769421078674 \n",
      "F1-score:  0.7480990924699534\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.7347302719572002 \n",
      "Recall:  0.7514819881440948 \n",
      "F1-score:  0.7430117222723174\n",
      "875 875 875 875 875\n"
     ]
    }
   ],
   "source": [
    "if('dataset' not in golddf.columns):\n",
    "    golddf['dataset'] = ds\n",
    "\n",
    "for dataset in golddf['dataset'].unique():\n",
    "    \n",
    "    pdfs=[]\n",
    "    print('------------------------------------------')\n",
    "    print('dataset: ',dataset)\n",
    "    \n",
    "    df1 = golddf[golddf['dataset']==dataset]\n",
    "    if('subtask' not in df1.columns):\n",
    "        df1['subtask'] = ['EV-EX']*len(df1)\n",
    "    for task in df1['subtask'].unique():\n",
    "        print('------------------------------------------')\n",
    "        print('subtask: ',task)\n",
    "        \n",
    "        # get list of indices and mask on pred\n",
    "        \n",
    "        #print(df2.index)\n",
    "        #print(len(df2),len(pred[df2.index]))\n",
    "        df2 = df1[df1['subtask']==task] # GOLD\n",
    "        g = df2[relcol]\n",
    "        ds = df2['dataset']\n",
    "        txt = df2['text']\n",
    "        \n",
    "        p = pd.Series(pred)[df2.index]\n",
    "        \n",
    "        print(len(g),len(p))\n",
    "    \n",
    "        #print(g,p)\n",
    "        posind,cor,_,_,_ = scoring(g,p,indices=df2.index,task=task,skipneg=True)\n",
    "        \n",
    "        posind1,cor1,_,_,_ = scoring(g,p,indices=df2.index,task=task,skipneg=False)\n",
    "        \n",
    "        ofile = 'results/'+predfolder+'/'+task+'_'+dataset+'.csv'\n",
    "        print(len(p),len(g),len(cor),len(ds),len(txt))\n",
    "        \n",
    "        \n",
    "        pdf = pd.DataFrame(data={'pred':p,'gold':g,'correct':cor,'dataset':ds,'text':txt})\n",
    "        \n",
    "        if(use_sent_mask):\n",
    "            pdf['pred_unmask'] = pd.Series(pred_o)[df2.index]\n",
    "        pdfs.append(pdf)\n",
    "        \n",
    "           \n",
    "    \n",
    "        \n",
    "        \n",
    "        '''\n",
    "        if(task=='EV-CLF'):\n",
    "            ct=0\n",
    "            for i,r in pdf.iterrows():\n",
    "                ct+='.'.join(r['pred'].split('.')[0:-1]) == '.'.join(r['gold'][0].split('.')[0:-1])\n",
    "                #print('.'.join(r['pred'].split('.')[0:-1]))\n",
    "                #print('.'.join(r['gold'][0].split('.')[0:-1]))\n",
    "            print(ct/len(pdf))\n",
    "        '''\n",
    "        #pdf = pd.concat(pdfs)\n",
    "        pdf.to_csv(ofile,index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "written-scenario",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'multitask4mixbidspec2verbose2_prin_ml_1024_50_4bs_2DAYS'"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predfolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dramatic-building",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "actual-clothing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(875, 875)"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(txt),txt.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "engaging-ranch",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "703"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([p for p in g if 'NONE' not in p])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "impressed-passion",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "sharing-heather",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_res = pd.read_json('results/'+predfolder+'/train_results.json',typ='series')\n",
    "state = pd.read_json('results/'+predfolder+'/trainer_state.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "blond-coordination",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>loss</th>\n",
       "      <th>step</th>\n",
       "      <th>total_flos</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_runtime</th>\n",
       "      <th>train_samples_per_second</th>\n",
       "      <th>train_steps_per_second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.52</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.4011</td>\n",
       "      <td>500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.04</td>\n",
       "      <td>0.000047</td>\n",
       "      <td>0.2306</td>\n",
       "      <td>1000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.56</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>0.1791</td>\n",
       "      <td>1500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.08</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>0.1527</td>\n",
       "      <td>2000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.60</td>\n",
       "      <td>0.000042</td>\n",
       "      <td>0.1273</td>\n",
       "      <td>2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  learning_rate    loss  step  total_flos  train_loss  train_runtime  \\\n",
       "0   1.52       0.000048  0.4011   500         NaN         NaN            NaN   \n",
       "1   3.04       0.000047  0.2306  1000         NaN         NaN            NaN   \n",
       "2   4.56       0.000045  0.1791  1500         NaN         NaN            NaN   \n",
       "3   6.08       0.000044  0.1527  2000         NaN         NaN            NaN   \n",
       "4   7.60       0.000042  0.1273  2500         NaN         NaN            NaN   \n",
       "\n",
       "   train_samples_per_second  train_steps_per_second  \n",
       "0                       NaN                     NaN  \n",
       "1                       NaN                     NaN  \n",
       "2                       NaN                     NaN  \n",
       "3                       NaN                     NaN  \n",
       "4                       NaN                     NaN  "
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epwiseloss = pd.DataFrame(list(state['log_history']))\n",
    "epwiseloss.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "affected-armor",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multitask4mixbidspec2verbose2_prin_ml_1024_50_4bs_2DAYS\n",
      "{'epoch': 50.0, 'step': 16450, 'total_flos': 1.102800094182604e+17, 'train_loss': 0.067719119022682, 'train_runtime': 10003.1839, 'train_samples_per_second': 26.302, 'train_steps_per_second': 1.6440000000000001}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+0AAAG0CAYAAABHWcYFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABovUlEQVR4nO3dd5hV1b3/8feaxhRm6GUUFFSQooKKJRbEjrGX2GtU9CYm3pjkF01yb4zp3eQmJmKvEWLsvYsaC6jYKIoIAoJSBIYZpq/fH+eAA1IGmJl9Zub9ep55zjl7r73Pd3ZOcD5nrb1WiDEiSZIkSZIyT1bSBUiSJEmSpHUztEuSJEmSlKEM7ZIkSZIkZShDuyRJkiRJGcrQLkmSJElShjK0S5IkSZKUoQztkiRJkqQ2L4RwYwjhsxDCu010vroQwuT0zwNNcc51vo/rtEuSJEmS2roQwkhgBXBrjHGnJjjfihhjxy2vbMPsaZckSZIktXkxxgnAkobbQgjbhxAeCyG8HkJ4IYQwKKHy1svQLkmSJElqr8YC34ox7g58D7hmE47NDyFMCiG8EkI4rlmqA3Ka68SSJEmSJGWqEEJHYB/gXyGEVZs7pPedAFy1jsPmxRgPTz/fNsY4L4SwHfBMCOGdGOOHTV2noV2SJEmS1B5lAUtjjMPX3hFjvAe4Z0MHxxjnpR9nhhCeA3YFmjy0OzxekiRJktTuxBiXAx+FEL4GEFKGNebYEEKXEMKqXvnuwL7AlOao09AuSZIkSWrzQgj/BF4GdgwhzA0hnA+cAZwfQngLeA84tpGnGwxMSh/3LPDrGGOzhHaXfJMkSZIkKUPZ0y5JkiRJUobKuInosrKyYkFBQdJlSJIkSZK0ySoqKmKMsck6yDMutBcUFFBeXp50GZIkSZIkbbIQwsqmPJ/D4yVJkiRJylAZ19MuSZIkSVJbFkKYBZQBdUBtjHHE+toa2iVJkiRJankHxhgXbayRoV2SJEmS1qGmpoa5c+dSWVmZdCnKQPn5+fTp04fc3NxmfZ+MW6e9qKgoOhGdJEmSpKR99NFHFBcX061bN0IISZejDBJjZPHixZSVldG/f/819oUQqoF3GmwaG2Mcu1abj4DPgQhcu/b+huxplyRJkqR1qKyspF+/fgZ2fUkIgW7durFw4cJ17d7gPepp+8UY54UQegJPhhCmxRgnrKuhs8dLkiRJ0noY2LU+W/LZiDHOSz9+BtwL7Lm+toZ2SZIkSZJaSAihKIRQvOo5cBjw7vraOzxekiRJkjJUx44dWbFiRdJlqGn1Au5N99TnAHfGGB9bX2NDuyRJkiRJLSTGOBMY1tj2Do+XJEmSpAwXY+T73/8+O+20EzvvvDPjxo0DYP78+YwcOZLhw4ez00478cILL1BXV8e55567uu2f/vSnhKvXlrCnXZIkSZI24qcPvseUT5Y36TmHbFXCT44e2qi299xzD5MnT+att95i0aJF7LHHHowcOZI777yTww8/nB/96EfU1dVRUVHB5MmTmTdvHu++m7pNeunSpU1at1qWPe2SJEmSlOFefPFFTjvtNLKzs+nVqxcHHHAAEydOZI899uCmm27iyiuv5J133qG4uJjtttuOmTNn8q1vfYvHHnuMkpKSpMvXFmhUT3sIYTTwZyAbuD7G+Ov1tDsRuBvYI8Y4Kb3tCuB8oA74dozx8aYoXJIkSZJaSmN7xFvayJEjmTBhAg8//DDnnnsul112GWeffTZvvfUWjz/+OP/4xz8YP348N954Y9KlajNttKc9hJAN/A04AhgCnBZCGLKOdsXApcCrDbYNAU4FhgKjgWvS52t5s1+GGJvsdDV19UxbsJzllTVNdk5JkiRJWpf999+fcePGUVdXx8KFC5kwYQJ77rkns2fPplevXlx44YVccMEFvPHGGyxatIj6+npOPPFEfv7zn/PGG28kXb62QGOGx+8JzIgxzowxVgN3Aceuo93PgN8AlQ22HQvcFWOsijF+BMxgA4vGN5t5r8NNo2HcmVDZNPehvPfJckZf/QL/mbG4Sc4nSZIkSetz/PHHs8suuzBs2DAOOuggfvvb39K7d2+ee+45hg0bxq677sq4ceO49NJLmTdvHqNGjWL48OGceeaZ/OpXv0q6fG2BxgyP3xqY0+D1XGCvhg1CCLsBfWOMD4cQvr/Wsa+sdezWm1nr5ttqNzjsF/Dk/8J1B8Ipt0PPwVt0yoG9OhICTFuwnNE79W6iQiVJkiTpC6vWaA8h8Lvf/Y7f/e53a+w/55xzOOecc750nL3rbccWT0QXQsgC/gh8dwvOMSaEMCmEMKm2tnZLS1rXG8A+l8A5D6Z62q87CN65e4tOWZiXQ/9uRUyd37QzSEqSJEmStEpjQvs8oG+D133S21YpBnYCngshzAL2Bh4IIYxoxLEAxBjHxhhHxBhH5OQ04yp0/faFiyZA753h3+fDY1dA3ebfkz64tISp88uasEBJkiRJkr7QmNA+ERgQQugfQsgjNbHcA6t2xhiXxRi7xxj7xRj7kRoOf0x69vgHgFNDCB1CCP2BAcBrTf5bbIqSUjjnIdjrYnjlGrjlaChbsFmnGtS7mI+XVLCiqhlGB0iSJEmS2r2NhvYYYy1wCfA4MBUYH2N8L4RwVQjhmI0c+x4wHpgCPAZ8M8ZYt+Vlb6GcPDjiN3DiDTD/Lbh2JMz+zyafZnBpar3D6QscIi9JkiRJanqNuqc9xvhIjHFgjHH7GOMv0tv+N8b4wDrajlq1Rnv69S/Sx+0YY3y06UpvAjufBBc8DXkd4eaj4OVrNmlZuEGlxQAOkZckSZIkNYstnoiu1es1BMY8CwNHw+NXwN1fh6oVjTp0684FFOfnOBmdJEmSJKlZGNoB8julloE7+Ccw5T64/mBY9MFGDwshMLh3iaFdkiRJktQsDO2rZGXB/pfBWfdC+UIYeyBM+dLo/y8ZXFrM9AVl1Nc3fli9JEmSJDVGx44dm/09/vGPf3Drrbc2+/usy80338wnn3yyycddffXV6625Mb/PzTffzCWXXLLOfb/85S9XP6+urmbkyJE0y9LkjWRoX9t2o1LLwvUYCOPPgif/F+rW/z/Q4NISyqvrmPN5RcvVKEmSJEmboK5u/fOBX3zxxZx99tmJvPfmhPba2lpuvPFGTj/99HXu29Lfp2Foz8vL4+CDD2bcuHGbfb4t1YyLordinfrAeY/CY5fDS3+GeW/ASTdBxx5fajooPYP81PllbNutqKUrlSRJktQSHr0cFrzTtOfsvTMc8etGN//d737H+PHjqaqq4vjjj+enP/0pAMcddxxz5syhsrKSSy+9lDFjxgCpXvqLLrqIp556ir/97W+MHj2aSy+9lIceeoiCggLuv/9+evXqxZVXXknHjh353ve+x6hRo9hrr7149tlnWbp0KTfccAP7778/FRUVnHvuubz77rvsuOOOfPLJJ/ztb39jxIgR66x17fd+5plnePDBB1m5ciX77LMP1157Lf/+97+ZNGkSZ5xxBgUFBbz88stMmTKFyy67jBUrVtC9e3duvvlmSktL1zj3M888w2677UZOTirOjho1iuHDh/Piiy9y2mmnUVZWtvr3mThxIueffz5ZWVkceuihPProo7z77rsAfPLJJ4wePZoPP/yQ448/nt/+9rdcfvnlrFy5kuHDhzN06FDuuOMOjjvuOK644grOOOOMTf6fuCnY074+OR3gqD/BsdfA3ImpZeHmTPxSsx17FRMC3tcuSZIkqdk88cQTfPDBB7z22mtMnjyZ119/nQkTJgBw44038vrrrzNp0iT+8pe/sHjxYgDKy8vZa6+9eOutt9hvv/0oLy9n77335q233mLkyJFcd91163yv2tpaXnvtNa6++urVXwxcc801dOnShSlTpvCzn/2M119/fYP1rv3el1xyCRMnTuTdd99l5cqVPPTQQ5x00kmMGDGCO+64g8mTJ5OTk8O3vvUt7r77bl5//XW+/vWv86Mf/ehL537ppZfYfffd19hWXV3NpEmT+O53v7vG9vPOO49rr72WyZMnk52dvca+yZMnM27cON555x3GjRvHnDlz+PWvf01BQQGTJ0/mjjvuAGCnnXZi4sQvZ8GWYk/7xux6RuobsHFnwk1HwOhfwR4XQAgAFORl079bEdNcq12SJElquzahR7w5PPHEEzzxxBPsuuuuAKxYsYIPPviAkSNH8pe//IV7770XgDlz5vDBBx/QrVs3srOzOfHEE1efIy8vj6OOOgqA3XffnSeffHKd73XCCSesbjNr1iwAXnzxRS699FIgFWJ32WWXDda79ns/++yz/Pa3v6WiooIlS5YwdOhQjj766DWOmT59Ou+++y6HHnookBpWv3YvO8D8+fMZPHjwGttOOeWUL7VbunQpZWVlfOUrXwHg9NNP56GHHlq9/+CDD6ZTp04ADBkyhNmzZ9O3b991/i55eXmUlZVRXFy8wd+7ORjaG6N0F7joebhnDDzyPZg7KdULn1cIpO5rf2fesoSLlCRJktRWxRi54ooruOiii9bY/txzz/HUU0/x8ssvU1hYyKhRo6isrAQgPz9/jd7l3NxcQrrzMTs7e72Tq3Xo0GGjbTam4XtXVlbyjW98g0mTJtG3b1+uvPLK1TWu/TsOHTqUl19+eYPnLigo+NLxRUWbfqvyqt8TNv67VlVVkZ+fv8nv0RQcHt9YBV3gtHEw6ofw9ji44VBYMhNIzSD/8ZIKyiprEi5SkiRJUlt0+OGHc+ONN7JixQoA5s2bx2effcayZcvo0qULhYWFTJs2jVdeeaVZ3n/fffdl/PjxAEyZMoV33mn8/f2rAnb37t1ZsWIFd9999+p9xcXFlJWVAbDjjjuycOHC1aG9pqaG995770vnGzx4MDNmzNjo+3bu3Jni4mJeffVVAO66665G1Zubm0tNzRfZbvHixXTv3p3c3NxGHd/UDO2bIisLRv0Azrgbls2Fa0fB9EcZ1Ds1Gd37n5YlW58kSZKkNumwww7j9NNP5ytf+Qo777wzJ510EmVlZYwePZra2loGDx7M5Zdfzt57790s7/+Nb3yDhQsXMmTIEH784x8zdOjQ1UPLN6Zz585ceOGF7LTTThx++OHsscceq/ede+65XHzxxQwfPpy6ujruvvtufvCDHzBs2DCGDx/Of/7zny+d74gjjlh9P//G3HDDDVx44YUMHz6c8vLyRtU8ZswYdtlll9UTzz377LMceeSRjXq/5hBizKz1xYuKimJ5eXnSZWzc57Ng/Nkw/y2W7/kdhk/YnZ8etwtn7b1t0pVJkiRJagJTp0790r3T7VVdXR01NTXk5+fz4YcfcsghhzB9+nTy8vISqWfVbO8DBgzYYLsVK1asXuv+17/+NfPnz+fPf/7zJr3XCSecwK9//WsGDhz4pX3r+oyEECpijE22tJj3tG+uLv3g60/AI9+l5LU/cVv+MJ6d80swtEuSJElqYyoqKjjwwAOpqakhxsg111yTWGCHLwL4xkL7ww8/zK9+9Stqa2vZdtttufnmmzfpfaqrqznuuOPWGdhbij3tTeH1W6h58Lt8ntWFnuePg613S7oiSZIkSVvInvaN22uvvaiqqlpj22233cbOO++cUEUty5721mL3c7hxeiFHT7+ceOPhhK/+HnY/J+mqJEmSJG2hGOPqGdf1ZasmeWuPWqoD3InomkinHfbiyKqfU7nV3vDgt+H+S6Dmy8sYSJIkSWod8vPzWbx4cYuFM7UeMUYWL17cIsvA2dPeRAaXlvA5JTy/598Z3f9mmPA7WPAOnHwrdPE+d0mSJKm16dOnD3PnzmXhwoVJl6IMlJ+fT58+fZr9fQztTWRgr2KyAkxdUMHoQ38MW+8O91wEYw+AE6+HHQ5JukRJkiRJmyA3N5f+/fsnXYbaOYfHN5GCvGz6dS9i6vzlqQ07HgFjnoWSreH2k+D530J9fbJFSpIkSZJaFUN7Exrcu4RpC8q+2NBtezj/SdjlZHj2F/DPU2Hl58kVKEmSJElqVQztTWhwaTEfL6mgrLLmi415hXD8tfDV38OHz8DYUal73SVJkiRJ2ghDexMaXFoCwPSGve0AIcCeF8J5j0JtNVx/CEz+ZwIVSpIkSZJaE0N7ExqUDu1T1w7tq/TdAy6aAH32gPsuhocug9qqFqxQkiRJktSaGNqb0Fad8inJz/liMrp16dgDzroP9r0UJt0ANx0By+a2WI2SJEmSpNbD0N6EQggMKi1h2oZCO0B2Dhx6VWoN94Xvw7UjYeZzLVKjJEmSJKn1MLQ3sSGlqRnk6+tjIxofm1oWrqgH3HY8vPgniI04TpIkSZLULhjam9ig3sVUVNcx5/OKxh3QfQBc8DQMOQ6euhLGnQmVy5qzREmSJElSK2Fob2KrZpDf4H3ta+vQEU66EQ7/FUx/FMYeCJ9OaaYKJUmSJEmthaG9iQ3sVUxWgCnz1zOD/PqEAF/5Bpz7EFSvgOsPhnfubp4iJUmSJEmtgqG9iRXkZdOve9HGJ6Nbn233SS0LVzoM/n0+PHo51NU0bZGSJEmSpFbB0N4MBpeWMHXBZoZ2gOLecM6DsPc34NW/w81HQdmCpitQkiRJktQqGNqbweDexcxZspKyyi3oIc/OhdG/ghNvgAVvwz/2h1kvNV2RkiRJkqSMZ2hvBqsmo5u+YBPva1+XnU+CC5+B/BK45Wh4+W8uCydJkiRJ7YShvRmsnkG+KUI7QM/BcOGzsOMR8PgP4e7zoGpF05xbkiRJkpSxDO3NoLRTPiX5OZu27NvG5JfAKbfDIT+FKffDdQfBwveb7vySJEmSpIxjaG8GIYTUZHRNGdpTJ4b9/hvOug8qFsN1B6YCvCRJkiSpTTK0N5PBpSVMX1BGfX0z3H++3QFw0fPQYxCMPxue+B+oq23695EkSZIkJapRoT2EMDqEMD2EMCOEcPk69l8cQngnhDA5hPBiCGFIenu/EMLK9PbJIYR/NPUvkKkGlxZTUV3Hx0sqmucNOvWB8x6BPS6A//wFbjsOVnzWPO8lSZIkSUrERkN7CCEb+BtwBDAEOG1VKG/gzhjjzjHG4cBvgT822PdhjHF4+ufiJqo74w3qnZqMbtqWrNe+MTkd4Mg/wHH/gLkT4doDYM5rzfd+kiRJkqQW1Zie9j2BGTHGmTHGauAu4NiGDWKMDZNpEdDu1yTbsXcxWQGmzG+iGeQ3ZPhpcP6TkJMHN30VXrvOZeEkSZIkqQ1oTGjfGpjT4PXc9LY1hBC+GUL4kFRP+7cb7OofQngzhPB8CGH/db1BCGFMCGFSCGFSbW3buDc7Pzeb/t2Lmn4yuvUp3QXGPAfbHwSPfA/uvQiqm2loviRJkiSpRTTZRHQxxr/FGLcHfgD8OL15PrBNjHFX4DLgzhBCyTqOHRtjHBFjHJGTk9NUJSVuUGlJ8w6PX1tBFzjtLjjwx/D2eLjhUFj8Ycu9vyRJkiSpSTUmtM8D+jZ43Se9bX3uAo4DiDFWxRgXp5+/DnwIDNysSluhIaUlzFmykrLKmpZ706wsOOD7cObdsHwejD0Qpj/acu8vSZIkSWoyjQntE4EBIYT+IYQ84FTggYYNQggDGrw8Evggvb1HeiI7QgjbAQOAmU1ReGswqHcxANMXtMB97Wvb4RAY8zx07Qf/PBWe/hnU17V8HZIkSZKkzbbR0B5jrAUuAR4HpgLjY4zvhRCuCiEck252SQjhvRDCZFLD4M9Jbx8JvJ3efjdwcYxxSRP/DhlrcGnqToAWu699bV22ha8/AbueBS/8Hm4/EcoXJ1OLJEmSJGmThZhhs4wXFRXF8vLypMtoEjFGhl/1JEfuUsovj9852WJevwUe+T507Akn3wpb75ZsPZIkSZLUBoUQKmKMRU11viabiE5fFkJgUO/i5HraG9r9HDj/cSDAjYenQrwkSZIkKaMZ2pvZ4NISpi8oo74+A0Y0bLUrXPQ89NsfHvw23P9NqFmZdFWSJEmSpPUwtDezwaXFVFTX8fGSDFkzvbArnPEvGPn/4M3bU73un89OuipJkiRJ0joY2ptZ4pPRrUtWNhz0IzhtHCyZBWMPgA+eSroqSZIkSdJaDO3NbGCvYrICTE1i2beN2XE0XPQclPSBO06C538L9fVJVyVJkiRJSjO0N7P83Gz6dy/KrJ72hrpuB+c/AbucAs/+IrWm+8rPk65KkiRJkoShvUUMLi3J3NAOkFcIx/8DjvwDfPgMXHsAzH8r6aokSZIkqd0ztLeAwaUlzP18Jcsra5IuZf1CgD0ugPMehboauOEwmHxn0lVJkiRJUrtmaG8Bg0uLAZieife1r63vHnDRBOizB9z3X/DQd6C2KumqJEmSJKnNCCFkhxDeDCE8tLG2hvYWMKh3agb5aZk8RL6hjj3grPtg3/+GSTfCTUfAsrlJVyVJkiRJbcWlwNTGNDS0t4DSTvl0KshlyvxW0NO+SnYOHPpTOPk2WPg+XDsSZj6XdFWSJEmS1KqFEPoARwLXN6a9ob0FhBAYXFrMtAWtpKe9oSHHwJhnoagH3HY8vPAHl4WTJEmSpPXLCSFMavAzZq39VwP/D2hUsDK0t5BBvUuYvqCM+vqYdCmbrvsAuOBpGHIcPH0VjDsTKpclXZUkSZIkZaLaGOOIBj9jV+0IIRwFfBZjfL2xJzO0t5AhpSVUVNcxe0lF0qVsng4d4aQbYfSv4YPHYewo+PS9pKuSJEmSpNZkX+CYEMIs4C7goBDC7Rs6wNDeQgalZ5BvNZPRrUsIsPd/wTkPQnU5XH8IvP2vpKuSJEmSpFYhxnhFjLFPjLEfcCrwTIzxzA0dY2hvIQN7FZMVYGprDu2rbLtPalm40mFwzwXwyP+D2uqkq5IkSZKkNsfQ3kLyc7PZrkdHpraGtdobo7h3qsd972/Ca9fCLUfB8vlJVyVJkiRJrUKM8bkY41Eba2dob0GDehe3jZ72VbJzYfQvU/e6L3g3tSzcrBeTrkqSJEmS2gxDewsaXFrC3M9XsryyJulSmtZOJ8KFT0N+CdxyDPzn/yC2wlnyJUmSJCnDGNpb0OD0ZHTT28oQ+YZ6DoYLn4Udj4Anfgz/Ogeq2uDvKUmSJEktyNDeggaXlgBtZDK6dckvgVNuh0OvgqkPwnUHwcLpSVclSZIkSa2Wob0F9S7Jp1NBLlPnt+Ee6BBg30vhrPugYkkquL93X9JVSZIkSVKrZGhvQSEEBpe2scno1me7A1LLwvUYlBoq//iPoK426aokSZIkqVUxtLewwaUlTF9QRn19O5iordPWcN4jsMcF8PJf4dZjYcVnSVclSZIkSa2Gob2FDe5dwsqaOmYvqUi6lJaR0wGO/AMcfy3Mez21LNzHryZdlSRJkiS1Cob2FtbmJ6Nbn2GnwgVPpkL8zV+FV8e6LJwkSZIkbYShvYUN6NWRrADT2ltoB+i9M4x5DnY4BB79PtwzBqrLk65KkiRJkjKWob2F5edms12PjkxpyzPIb0hBFzj1n3Dgj+Gdf8H1h8DiD5OuSpIkSZIykqE9AYNLS5i2oB32tK+SlQUHfB/OvBvK5sPYUTDt4aSrkiRJkqSMY2hPwKDexcz9fCXLK2uSLiVZOxySWhau2/Zw1+nw9FVQX5d0VZIkSZKUMQztCRiSnoxu+oJ2OkS+oc7bwHmPwW7nwAt/gNtPgPLFSVclSZIkSRnB0J6AQaXFQDucQX59cvPhmL/AMf8Hs19OLQs39/Wkq5IkSZKkxBnaE9C7JJ/OhbmG9rXtdjac/ziELLhpNEy6yWXhJEmSJLVrhvYEhBAY1LuYqe11BvkN2WpXuOh56Lc/PPTfcP8lULMy6aokSZIkKRGG9oQMLi1h+oIy6urtSf6Swq5wxr/ggB/A5NvhhsPg81lJVyVJkiRJLc7QnpDBpSWsrKnj4yUVSZeSmbKy4cAfwunjYelsuPYA+ODJpKuSJEmSpBbVqNAeQhgdQpgeQpgRQrh8HfsvDiG8E0KYHEJ4MYQwpMG+K9LHTQ8hHN6Uxbdmg3unZpD3vvaNGHg4jHkeOvWFO74Gz/0a6uuTrkqSJEmSWsRGQ3sIIRv4G3AEMAQ4rWEoT7szxrhzjHE48Fvgj+ljhwCnAkOB0cA16fO1ewN6dSQrwDRD+8Z17Q/nPwHDToPnfgV3ngwVS5KuSpIkSZKaXWN62vcEZsQYZ8YYq4G7gGMbNogxNkyeRcCqG7WPBe6KMVbFGD8CZqTP1+7l52azXY+OTHEyusbJK4TjroEj/wgzn4Oxo2D+W0lXJUmSJEnNqjGhfWtgToPXc9Pb1hBC+GYI4UNSPe3f3sRjx4QQJoUQJtXW1ja29lZvcGmJw+M3RQiwx/nw9cegvjY1Qd2bdyRdlSRJkiQ1myabiC7G+LcY4/bAD4Afb+KxY2OMI2KMI3JycpqqpIw3uLSYeUtXsryyJulSWpc+I+CiCdB3T7j/G/DgpVBblXRVkiRJktTkGhPa5wF9G7zuk962PncBx23mse3KqsnopjlEftMVdYcz74X9vgOv3ww3joalczZ6mCRJkiS1Jo0J7ROBASGE/iGEPFITyz3QsEEIYUCDl0cCH6SfPwCcGkLoEELoDwwAXtvystuGwaXp0L7AIfKbJTsHDrkSTrkDFs+Aa0fCh88mXZUkSZIkNZmNhvYYYy1wCfA4MBUYH2N8L4RwVQjhmHSzS0II74UQJgOXAeekj30PGA9MAR4DvhljrGv6X6N16lXSgc6Fud7XvqUGHwUXPgsde8HtJ8CE37ssnCRJkqQ2IcQYN96qBRUVFcXy8vKky2gxp419hYqaOu7/5r5Jl9L6VZfDA9+Gd++GHb8Kx/0dCjonXZUkSZKkdiSEUBFjLGqq8zXZRHTaPINKi3l/QRl19Zn15UmrlFcEJ14Po38DHzwB1x0In76XdFWSJEmStNkM7QkbXFrCypo6Zi9uP6MLmlUIsPfFcO7DUF0B1x0Mb49PuipJkiRJ2iyG9oQNWT0ZnTPIN6lt9k4tC7f1bnDPhfDI96G2OumqJEmSJGmTGNoTtkPPjmRnBSejaw7FveDs++Erl8BrY+HmI2H5J0lXJUmSJEmNZmhPWH5uNtt1L2Kqa7U3j+xcOPwXcNJNqfvbrx0JH72QdFWSJEmS1CiG9gwwqLTEnvbmttMJcOEzkN8Zbj0WXvoLZNjKCZIkSZK0NkN7BhhcWsy8pStZtrIm6VLatp6DUsF90JHw5P/A+LOhyhEOkiRJkjKXoT0DDE5PRjfdyeiaX34JnHwrHPozmPYQXHcQLJyedFWSJEmStE6G9gwwuHcqtDtEvoWEAPt+OzVJ3crPU8H9vXuTrkqSJEmSvsTQngF6lXSgS2Eu0xYY2ltU/5GpZeF6DoF/nQuP/wjqapOuSpIkSZJWM7RngBACg3qXMMUZ5FteyVZw7sOw5xh4+a+pSerKPk26KkmSJEkCDO0ZY3BpCe8vKKOu3hnNW1xOHnz1d3D8WJj3empZuI9fSboqSZIkSTK0Z4pBpcWsrKlj9uLypEtpv4adAhc8BbkFcPOR8Oq1LgsnSZIkKVGG9gwxpHTVZHQOkU9U751gzHOww6Hw6P+Dey6Ear9IkSRJkpQMQ3uG2KFnR7KzgpPRZYKCznDqnXDQ/8A7d8P1h8DiD5OuSpIkSVI7ZGjPEPm52WzXvchl3zJFVhaM/B6cdQ+ULYCxo2Daw0lXJUmSJKmdMbRnkMGlJQ6PzzTbHwQXPQ/dtoe7Toenfgr1dUlXJUmSJKmdMLRnkEGlxcxbupJlK2uSLkUNdd4GznsMdj8XXvwj3H4ClC9KuipJkiRJ7YChPYMMTk9GN80h8pknNx+O/jMc81eY/TJcewDMfT3pqiRJkiS1cYb2DLJqBvlpCxwin7F2OwvOfyJ1z/tNo2HSjS4LJ0mSJKnZGNozSM/iDnQpzHUyuky31XAY8zz0HwkPfQfu/ybUrEy6KkmSJEltkKE9g4QQUpPR2dOe+Qq7wunj4YDLYfIdcMOhsOSjpKuSJEmS1MYY2jPMoN4lTF+wnLp6h1xnvKxsOPCKVHhf+jGMPQDefyLpqiRJkiS1IYb2DDO4tJjKmnpmLy5PuhQ11sDDU8PlO20Dd34Nnv2ly8JJkiRJahKG9gyzagZ512tvZbr2T01QN+x0eP43cOfJULEk6aokSZIktXKG9gyzQ8+OZGcFJ6NrjfIK4bhr4Kg/wcznU8PlP5mcdFWSJEmSWjFDe4bJz81m+x5FTFtgaG+VQoARX4evP5YaIn/DYfDm7UlXJUmSJKmVMrRnoEG9Sxwe39r1GQEXTYBt9kotCffAt6GmMumqJEmSJLUyhvYMNLi0hHlLV7JsZU3SpWhLFHWHM++F/b4Db9wCN41OzTIvSZIkSY1kaM9Ag0qLAZjmfe2tX3YOHHIlnHonLP4Qrj0APnwm6aokSZIktRKG9gw0JD2D/LQFDpFvMwYdCWOeg+LecNsJMOH3UF+fdFWSJEmSMpyhPQP1LO5A16I8Z5Bva7ptDxc8BTufBM/8DO46HVYuTboqSZIkSRnM0J6BQggM6l1saG+L8orghOvgiN/CjCdh7ChY8G7SVUmSJEnKUIb2DDW4tITpn5ZRVx+TLkVNLQTY6yI492GoWQnXHwJvjUu6KkmSJEkZyNCeoQb1Lqaypp5Zi8uTLkXNZZu9U8vCbb0b3DsGHvk+1FYnXZUkSZKkDGJoz1CDV01G53rtbVtxLzj7fvjKJfDaWLj5SFj+SdJVSZIkScoQjQrtIYTRIYTpIYQZIYTL17H/shDClBDC2yGEp0MI2zbYVxdCmJz+eaApi2/LBvTqSHZW8L729iA7Fw7/BXztZvj0Pbh2JHz0QtJVSZIkSWoGIYT8EMJrIYS3QgjvhRB+uqH2Gw3tIYRs4G/AEcAQ4LQQwpC1mr0JjIgx7gLcDfy2wb6VMcbh6Z9jNum3acc65GSzfY8iQ3t7MvR4GPMsFHSBW4+Fl/4C0TkNJEmSpDamCjgoxjgMGA6MDiHsvb7Gjelp3xOYEWOcGWOsBu4Cjm3YIMb4bIyxIv3yFaDP5lSuNQ0uLXGt9vamx45w4TMw+Ch48n9g/NlQ5WdAkiRJaitiyor0y9z0z3p76xoT2rcG5jR4PTe9bX3OBx5t8Do/hDAphPBKCOG4dR0QQhiTbjOptra2ESW1D4N6lzBv6UqWVdQkXYpaUodi+NotcNjPYdrDMPZA+Gxa0lVJkiRJapycVfk2/TNm7QYhhOwQwmTgM+DJGOOr6ztZk05EF0I4ExgB/K7B5m1jjCOA04GrQwjbr31cjHFsjHFEjHFETk5OU5bUqg0uLQZg2gKHyLc7IcA+30pNUle5FK47CN69J+mqJEmSJG1c7ap8m/4Zu3aDGGNdjHE4qVHqe4YQdlrfyRoT2ucBfRu87pPetoYQwiHAj4BjYoxVDYqZl36cCTwH7NqI9xRfzCDvfe3tWP/9U8vC9RoKd58Hj/0Q6hx5IUmSJLUFMcalwLPA6PW1aUxonwgMCCH0DyHkAacCa8wCH0LYFbiWVGD/rMH2LiGEDunn3YF9gSmb+Hu0Wz2LO9C1KM/72tu7kq3g3Idhz4vglb/BLcdA2adJVyVJkiRpM4QQeoQQOqefFwCHAuu9H3ajoT3GWAtcAjwOTAXGxxjfCyFcFUJYNRv874COwL/WWtptMDAphPAWqW8Pfh1jNLQ3UgiBwaXF9rQLcvLgq7+FE66DT95MLQs3++Wkq5IkSZK06UqBZ0MIb5PqJH8yxvjQ+hqHmGFLShUVFcXy8vKky8gYP3toCne8Opv3fjqa7KyQdDnKBAvehfFnwdKPU5PV7XVx6h54SZIkSYkLIVTEGIua6nxNOhGdmt7g0hIqa+qZtdgvMpTWeye48FkYcBg8djn8+3yoWrHx4yRJkiS1Oob2DDeod2oGeYfIaw0FneGUO+Dg/4X37oXrD4FFM5KuSpIkSVITM7RnuAG9OpKTFZg238notJasLNj/u3DmPVD+GYwdBVMfTLoqSZIkSU3I0J7hOuRks32Pjva0a/22PxDGPA/dB8C4M+HJn0BdbdJVSZIkSWoChvZWYFBpscu+acM694WvPwa7nwsvXQ23Hw8rFiZdlSRJkqQtZGhvBQaXljBv6UqWVdQkXYoyWU4HOPrPcOw1MOc1GHsAzJ2UdFWSJEmStoChvRVYPRndAofIqxF2PQPOfwKysuHG0TDxBsiwpR0lSZIkNY6hvRUYUloCwDTva1djlQ5L3ee+3QHw8GVw339BdUXSVUmSJEnaRIb2VqBHcQe6FuUx1RnktSkKu8Lp/4IDLoe37oIbDoMlM5OuSpIkSdImMLS3AiEEBpcWM83h8dpUWVlw4BVw+nhY9nFqWbj3H0+6KkmSJEmNZGhvJQb3LmH6p2XU1XtvsjbDwMNSw+U7bwN3ngzP/hLq65KuSpIkSdJGGNpbiZ227kRlTT0vfOAyXtpMXfvD+U/CsNPh+d/AHV+DiiVJVyVJkiRpAwztrcQRO/emX7dCrnpwClW19pBqM+UWwHHXwFF/glkvwLUHwCdvJl2VJEmSpPUwtLcSHXKyufKYocxcVM71L3yUdDlqzUKAEV+H8x6DWA83HA5v3JZ0VZIkSZLWwdDeiozasSejh/bm/575gHlLVyZdjlq7PrvDRc/DNnvDA5fAA9+Cmsqkq5IkSZLUgKG9lfmfo4cAcNWD7yVcidqEou5w1r2w32Xwxq1w4+Gw9OOkq5IkSZKUZmhvZbbuXMC3DhrA4+99yrPTP0u6HLUFWdlwyE/g1DtT67hfOxJmPJ10VZIkSZIwtLdKF+6/Hdv1KOLKB96jssZJ6dREBh0JY56D4lK4/USY8Duor0+6KkmSJKldM7S3Qnk5Wfz0mKHMXlzBdRNmJl2O2pJu28MFT8HOJ8EzP4e7ToeVS5OuSpIkSWq3DO2t1P4DenDkzqX89dkZzFlSkXQ5akvyiuCE6+CI38GMJ2HsKFjwbtJVSZIkSe2Sob0V+/FRg8nOCvz0wSlJl6K2JgTYawyc+wjUVsL1h8Bb45KuSpIkSWp3DO2tWGmnAi49eABPTf2Up6d+mnQ5aou22QvGPA9b7w73joGHvwe11UlXJUmSJLUbhvZW7rx9+7NDz45c+aCT0qmZFPeCs++Hfb4FE6+Dm78Ky+YlXZUkSZLULhjaW7m8nCyuOmYoc5as5O/PfZh0OWqrsnPgsJ/D126Bz6bC2APgowlJVyVJkiS1eYb2NmCfHbpz9LCt+PvzHzJ7cXnS5agtG3ocXPgMFHSFW4+Fl/4MMSZdlSRJktRmGdrbiB99dTC5WYErH3iPaIhSc+qxI1z4NAw+Bp78Xxh/FlQuT7oqSZIkqU0ytLcRvTvl851DB/Ls9IU8NfWzpMtRW9ehGL52Mxz2C5j2CFx3EHw2LemqJEmSpDbH0N6GnLNPPwb26siVD7zHymonpVMzCwH2uQTOeRAql6WC+7v3JF2VJEmS1KYY2tuQ3Owsrjp2J+YtXck1z81Iuhy1F/32hYsmQO+d4O7z4LEfQl1N0lVJkiRJbYKhvY3Ze7tuHDd8K659fiYfLXJSOrWQklI45yHY62J45W9wyzFQtiDpqiRJkqRWz9DeBv3wyMF0yMniJ05Kp5aUkwdH/AZOvAHmT4ZrR8Lsl5OuSpIkSWrVDO1tUM/i1KR0E95fyOPvfZp0OWpvdj4JLnga8jrCLUfBK393WThJkiRpMxna26izv7Itg3oXc9WD71FRXZt0OWpveg2BMc/CgMPhscvh7q9D1Yqkq5IkSZJaHUN7G5WTnpTuk2WV/PUZJ6VTAvI7wSm3w8E/gSn3wfUHw6IPkq5KkiRJalUM7W3Ynv27csJuW3PdCzP5cKG9nEpAVhbsfxmcdS+UL4SxB8LUB5OuSpIkSWo1DO1t3BVHDCY/N5srnZROSdpuVGpZuB4DYdyZ8ORPoM7bNiRJkqSNaVRoDyGMDiFMDyHMCCFcvo79l4UQpoQQ3g4hPB1C2LbBvnNCCB+kf85pyuK1cT2KO/C9w3bkhQ8W8ei7LsGlBHXqA+c9CiO+Di9dDbcdBysWJl2VJEmSlNHCxnpfQwjZwPvAocBcYCJwWoxxSoM2BwKvxhgrQgj/BYyKMZ4SQugKTAJGABF4Hdg9xvj5+t6vqKgolpe7vnhTqq2r55i/vsSS8mqe/u4BFHXISboktXeT74SHvgMFXeHkW6HvHklXJEmSJDWJEEJFjLGoqc7XmJ72PYEZMcaZMcZq4C7g2IYNYozPxhgr0i9fAfqknx8OPBljXJIO6k8Co5umdDVWTnYWPztuKAuWV/KXZ5wITBlg+Olw/pOQnQs3HQETr3dZOEmSJGkdGhPatwbmNHg9N71tfc4HHt2UY0MIY0IIk0IIk2prvc+1Oey+bVe+tnsfbnjhI2Z8VpZ0ORKU7gIXPQ/bHwgPfxfu+y+ortj4cZIkSVI70qQT0YUQziQ1FP53m3JcjHFsjHFEjHFETo5Dt5vL5UcMojAvm/+930nplCEKusBp42DUD+Gtu+CGQ2HJzKSrkiRJkjJGY0L7PKBvg9d90tvWEEI4BPgRcEyMsWpTjlXL6NaxA98fPYj/fLiYB9+en3Q5UkpWFoz6AZzxL1g2F64dBdMfS7oqSZIkKSM0JrRPBAaEEPqHEPKAU4EHGjYIIewKXEsqsH/WYNfjwGEhhC4hhC7AYeltSsjpe27DTluX8POHprCiylsRlEEGHJoaLt9lW/jnKfDML6C+LumqJEmSpERtNLTHGGuBS0iF7anA+BjjeyGEq0IIx6Sb/Q7oCPwrhDA5hPBA+tglwM9IBf+JwFXpbUpIdlbgZ8fuxMIVVfz5qfeTLkdaU5d+cP4TMPxMmPBbuONrUOE/GZIkSWq/NrrkW0tzybeWccU9bzN+0lwe+fb+7Ni7OOlypDXFCG/cAo98Hzr2hlNuha12TboqSZIkaaOSWPJNbdD3Dx9EcX4O/3v/u05Kp8wTAux+Lnz9MYj1cMPh8MatSVclSZIktThDezvVtSiP/3f4IF79aAn3T/4k6XKkddt6d7hoAmy7DzzwrdRPTWXSVUmSJEktxtDejp2yR1+G9enELx6ZyvLKmqTLkdatqBuc+W/Y/3up3vYbD4elHyddlSRJktQiDO3tWHZW4GfH7cSiFVVc/eQHSZcjrV9WNhz8P3DqP1PruF87EmY8lXRVkiRJUrMztLdzu/TpzOl7bsMtL89i6vzlSZcjbdigr8KY56B4K7j9JHj+d1Bfn3RVkiRJUrMxtIvvH74jJU5Kp9ai2/ZwwZOw89fg2Z/DXafByqVJVyVJkiQ1C0O76FyYx+VHDGLirM+55415SZcjbVxeEZwwFr76+9Qw+bGjYME7SVclSZIkNTlDuwD42u592XWbzvzq0aksW+mkdGoFQoA9L4RzH4HaSrj+UHjrrqSrkiRJkpqUoV0AZGUFfnbsTiwpr+ZPT76fdDlS422zV2pZuD4j4N6L4OHvQm110lVJkiRJTcLQrtV22roTZ+69Lbe+PIt35y1Luhyp8Tr2hLPug32+DROvh5u/Csu81UOSJEmtn6Fda/juoTvSpTCP/73/XerrnZROrUh2Dhz2Mzj5VvhsampZuI8mJF2VJEmStEUM7VpDp8JcLj9iEG98vJS735ibdDnSphtyLFz4LBR2g1uPhRevBldFkCRJUitlaNeXnLhbH3bftgs/f2gKr320JOlypE3XYyBc+EwqwD/1Exh/FlQuT7oqSZIkaZMZ2vUlWVmBq08ZTvfiDpxx/SvcY4+7WqMOHeGkm+DwX8K0R+C6A1PD5iVJkqRWJMQMGzZaVFQUy8vLky5DwLKKGi6+/XVenrmYbx+0A985dCAhhKTLkjbdrJfg7vOgagUc+3+w04lJVyRJkqQ2KoRQEWMsarLzGdq1IdW19fzPfe8ybtIcjh62Fb87aRfyc7OTLkvadGULYPw5MOcV2PsbcOhVkJ2bdFWSJElqYwztanExRq6dMJNfPzqN3bbpzNizR9C9Y4eky5I2XV0NPPE/8OrfYZuvwNduhuLeSVclSZKkNsTQrsQ8+s58vjN+Mt07duDGc/dgYK/ipEuSNs87d8MD34IOxangvu0+SVckSZKkNqKpQ7sT0anRjti5lHFjvkJVbT0nXvMfJry/MOmSpM2z80lwwdOQ1xFuPgpevsZl4SRJkpSR7GnXJpu3dCXn3zyRDz5bwVXHDuWMvbZNuiRp81Qug/u+AdMegqEnwDH/l5p1XpIkSdpMDo9XRlhRVcu37nyDZ6cv5IL9+nPFVweTneXM8mqFYoSXroanr4LuA+Hk21LrvEuSJEmbYWOhPYTQF7gV6AVEYGyM8c/rbW9o1+aqravn5w9P5eb/zOKQwb3486nDKeqQk3RZ0uaZ+Rzc/XWorYLjroEhxyZdkSRJklqhRoT2UqA0xvhGCKEYeB04LsY4ZV3tvaddmy0nO4srjxnKT48ZyjPTPuVr/3iZ+ctWJl2WtHm2GwUXTYAeO8L4s1OzzNfVJl2VJEmS2pgY4/wY4xvp52XAVGDr9bW3p11N4tlpn3HJnW/QMT+HG87Zg5227pR0SdLmqa2Cx66ASTdAv/3hpBuhY8+kq5IkSVIrEUKoBt5psGlsjHHsetr2AyYAO8UYl6+zjaFdTWXq/OWcf/NEPq+o4c+nDuewoa5/rVZs8p3w0HegoCucfAv03TPpiiRJktQKNHYiuhBCR+B54BcxxnvW187h8Woyg0tLuO+SfRnYqyMX3f46178wk0z7UkhqtOGnw/lPQnYu3PRVeO06l4WTJElSkwgh5AL/Bu7YUGAHe9rVDFZW13HZ+Mk8+u4CTt9rG356zFBys/1+SK3Uys/hnjHwwRMw7DQ46k+QW5B0VZIkScpQjZiILgC3AEtijP+90fMZ2tUc6usjv3tiOn9/7kP2H9Cdv56+G50KcpMuS9o89fXw/G/g+V9D753hlNuhS7+kq5IkSVIGakRo3w94gdR97/XpzT+MMT6yzvaGdjWn8ZPm8MN73qFf9yJuOncP+nYtTLokafO9/zjccyEQ4MQbYMAhSVckSZKkDNPYe9obyzHLalYnj+jLrefvycKyKo7720u8PvvzpEuSNt/Aw2HMc9CpD9xxEjz/u1QvvCRJktRM7GlXi/hw4Qq+fvNE5i+r5PdfG8Yxw7ZKuiRp81VXwIOXwjvjYeARcPw/oKBz0lVJkiQpAzR1T7uhXS1mSXk1F9/2Oq/NWsJlhw7kWwftQGoOBqkVihFeGwuP/xA6bwOn3AG9hiRdlSRJkhJmaFerVlVbxxX/fod73pzH8btuza9P3JkOOdlJlyVtvtkvw7/OgaoyOOb/YOeTkq5IkiRJCTK0q9WLMfLXZ2bwhyffZ49+Xbj2rBF0LcpLuixp85UtgH+dCx+/DHt/Ew79aWp9d0mSJLU7hna1GQ++9Qnf/ddblHbK58Zz92D7Hh2TLknafHU18MSP4dV/wLb7wtduho49k65KkiRJLczQrjbl9dmfM+bWSdTU1fOPM3dnnx26J12StGXeGpeapK6gM5x8K/TdM+mKJEmS1IISWfIthDA6hDA9hDAjhHD5OvaPDCG8EUKoDSGctNa+uhDC5PTPA01VuNqG3bftwn3f3JdeJfmcdeNr3PbyrKRLkrbMsFPggichpwPc9FV47brUpHWSJEnSZthoT3sIIRt4HzgUmAtMBE6LMU5p0KYfUAJ8D3ggxnh3g30rYoyNHvdsT3v7VFZZw6V3TeaZaZ9x+l7bcOXRQ8nLadR3SlJmWvk53HMRfPA47HIKHHU15BUmXZUkSZKaWRI97XsCM2KMM2OM1cBdwLENG8QYZ8UY3wbqm6owtS/F+blcd/YI/mvU9tz56secef2rLF5RlXRZ0uYr6AKn3QUH/hjeHg83HAqLP0y6KkmSJLUyjQntWwNzGryem97WWPkhhEkhhFdCCMetq0EIYUy6zaTa2tpNOLXakuyswA9GD+LPpw7nrblLOeavLzHlk+VJlyVtvqwsOOD7cObdsHwejD0Qpj+adFWSJElqRVpi/PG2McYRwOnA1SGE7dduEGMcG2McEWMckZOT0wIlKZMdO3xr/nXxV6irj5z49//w6Dvzky5J2jI7HAJjnoeu/eCfp8LTP4P6uqSrkiRJUivQmNA+D+jb4HWf9LZGiTHOSz/OBJ4Ddt2E+tRO7dKnMw9csi+DSov5rzve4E9Pvk99vZN5qRXrsi18/QnY9Sx44fdw+4lQvjjpqiRJkpThGhPaJwIDQgj9Qwh5wKlAo2aBDyF0CSF0SD/vDuwLTNnwUVJKz5J87hqzNyft3oc/P/0B/3XH65RXefuEWrHcfDj2r3D0X2D2f2DsATDvjaSrkiRJUgZr1DrtIYSvAlcD2cCNMcZfhBCuAibFGB8IIewB3At0ASqBBTHGoSGEfYBrSU1QlwVcHWO8YUPv5ezxWluMkRte/IhfPjKVgb2Kue7sEfTt6izcauXmvQHjz4YVn8JXfw+7n5N0RZIkSWoCTT17fKNCe0sytGt9Jry/kEvufIOc7CyuOWM39t6uW9IlSVumfDHccwF8+AzsemYqvOcWJF2VJEmStoChXe3aR4vKueCWicxeXMFPjhnKWXtvm3RJ0papr4PnfgUTfgelw+Dk21L3v0uSJKlVMrSr3VteWcOl/3yTZ6cv5Iy9tuEnRw8lL6clFkKQmtH0R+Gei1LLxJ1wPQw4JOmKJEmStBkM7RJQVx/53ePT+cfzH7JX/65cc8ZudOvYIemypC2z+MPUfe6fvgejroCR30+FeEmSJLUahnapgfvenMf/+/fb9OjYgevPGcHg0pKkS5K2THUFPPQdePsuGHA4nHAtFHRJuipJkiQ1kqFdWstbc5Yy5rZJlFXW8seThzF6p9KkS5K2TIww8Xp47ArotHXqPvfSXZKuSpIkSY3Q1KHdcZdq9Yb17cyDl+zHwF7FXHz7G1z91PvU12fWl1HSJgkB9rwQznsUaqvhhkNh8j+TrkqSJEkJsKddbUZlTR0/uvdd/v3GXI7YqTe//9owijrkJF2WtGVWLIS7z4NZL8CI82H0ryDH+RskSZIylcPjpQ2IMXLDix/xy0emMrBXMdedPYK+XQuTLkvaMnW18MxV8NKfYevd4eRboVOfpKuSJEnSOhjapUZ4/v2FXHLnG+RmZ3HNGbux93bdki5J2nJT7of7vgk5eXDSjbDdqKQrkiRJ0lq8p11qhAMG9uD+b+5L58Jczrz+Ve54dXbSJUlbbsixMOZZKOoBtx0PL/wxNWmdJEmS2ix72tWmLa+s4dv/fJPnpi/kzL234SdHDyU32++q1MpVrYAHvgXv3QODjoLjroH8TklXJUmSJBweL22yuvrIbx+bxrUTZrL3dl255ozd6VqUl3RZ0paJEV79BzzxY+i8LZxyO/QaknRVkiRJ7Z6hXdpM9745lx/8+x16FnfgurNHMLi0JOmSpC03+z/wr3OhqgyO+T/Y+aSkK5IkSWrXDO3SFpg8ZykX3TaJsspafnL0EE7crQ85DpdXa1e2IBXcP34Z9roYDv1ZarI6SZIktThDu7SFPl1eyTfveINJsz9n+x5FXHbojhyxU2+yskLSpUmbr64GnvxfeOUa6Ls3fO1mKClNuipJkqR2x9AuNYEYI4+9u4A/PPk+Mz5bwdCtSvje4TsyamAPQjC8qxV7999w/7cgrygV3Pvtm3RFkiRJ7YqhXWpCdfWR+96cx9VPv8+cJSvZo18XvnfYjuzluu5qzT6bCuPOhCUfwaFXwVe+CX4ZJUmS1CIM7VIzqK6tZ9ykOfzf0x/wWVkVIwf24PuH7cjOfVxGS61U5XK4779g2kMw5Dg49q/QoTjpqiRJkto8Q7vUjFZW13Hry7P4+/MfsrSihtFDe/PdwwYyoJdhR61QjPCfv8BTV0K3Aall4XoMTLoqSZKkNs3QLrWA5ZU13PDCR9zw4kdUVNdy3K5b898HD2SbboVJlyZtupnPw91fh9pKOO4aGHJs0hVJkiS1WYZ2qQUtKa/mH89/yC3/mUVdfeTUPfvyrYMG0KskP+nSpE2zbB6MPxvmTYJ9vgUHXwnZOUlXJUmS1OYY2qUELFhWyf898wHjJs4hOytwzj79uPiA7ela5FrYakVqq+DxH8LE66Hf/nDSjdCxZ9JVSZIktSmGdilBHy+u4Oqn3ufeyfMoysvhgv37c/5+/SnOz026NKnx3roLHvxvKOgMJ98KffdMuiJJkqQ2w9AuZYD3Py3jj0+8z2PvLaBLYS7/NWp7zv5KP/Jzs5MuTWqcBe+kloVbNg8O/yXseaHLwkmSJDUBQ7uUQd6as5TfPzGdFz5YRK+SDnzroAGcPKIveTlZSZcmbdzKz+Hei+H9x2Dnk+HoqyGvyf77IkmS1C4Z2qUM9MrMxfz+8elMmv0523Qt5L8PGcCxw7cmO8ueS2W4+np44Q/w7C+g5xA45Tbotn3SVUmSJLVahnYpQ8UYeW76Qn7/xHTe+2Q5A3p25LuH7cjhQ3sRHHasTDfjKfj3BakQf/w/YNBXk65IkiSpVTK0Sxmuvj7y6LsL+MOT05m5sJxd+nTie4ftyP4Duhveldk+n51aFm7+ZNj/u3DgjyDLeRokSZI2haFdaiVq6+q59815XP3UB8xbupJhfTtz8cjtOGxob4fNK3PVVMKj34c3boXtRsGJN0JRt6SrkiRJajUM7VIrU1Vbx/hJc7luwkw+XlJBv26FnL//dnxt9z7ONq/M9fot8Mj3oahHalm4PrsnXZEkSVKrYGiXWqm6+sjj7y3g2gkzeWvOUroW5XHOV/px1le2pWtRXtLlSV/2yZsw7mxYsQCO+C3sfq7LwkmSJG2EoV1q5WKMvPbREsZOmMnT0z4jPzeLk0f05YL9tmObboVJlyetqWJJaoK6D5+G4WfCkb+H3IKkq5IkScpYhnapDfng0zLGTpjJfZPnUVcfOWKnUsaM3I5hfTsnXZr0hfo6eP43qZ/eu6SWhevSL+mqJEmSMpKhXWqDPl1eyU0vzeKOV2dTVlnL3tt15aKR2zNqxx7OOK/MMf0xuHcMEODE62HAoUlXJEmSlHEM7VIbVlZZw7iJc7jhxY+Yv6ySgb06cuH+23Hs8K3Jy8lKujwJlsxM3ef+6bsw6nIY+f8gy8+mJEnSKoZ2qR2oqavnwbc+YeyEmUxbUEavkg6ct29/Tt9rG0ryc5MuT+1ddQU8fBm89U8YcBgcfy0Udk26KkmSpIyQSGgPIYwG/gxkA9fHGH+91v6RwNXALsCpMca7G+w7B/hx+uXPY4y3bOi9DO3SF2KMTPhgEWMnfMhLMxbTsUMOp++1Deft24/STk4GpgTFCJNuhEd/ACVbpe5zLx2WdFWSJEmJa/HQHkLIBt4HDgXmAhOB02KMUxq06QeUAN8DHlgV2kMIXYFJwAggAq8Du8cYP1/f+xnapXV7d94yrp0wk0femU8Ajhm+FWNGbseg3iVJl6b2bO4kGH82VCyGI/8Iu56RdEWSJEmJSiK0fwW4MsZ4ePr1FQAxxl+to+3NwEMNQvtpwKgY40Xp19cCz8UY/7m+9zO0Sxs2Z0kFN7z4EeMmzmFlTR2jduzBmJHb8ZXtujlpnZKxYiH8++vw0QTY/Tw44jeQ0yHpqiRJkhLR1KG9MbMHbQ3MafB6bnpbYzTq2BDCmBDCpBDCpNra2kaeWmqf+nYt5MpjhvLyFQfxvcMG8u68ZZx+3asc89eXePCtT6itq0+6RLU3HXvAmffCft+B12+CG0fD0jkbP06SJEkblRFT/sYYx8YYR8QYR+Tk5CRdjtQqdC7M45KDBvDiDw7il8fvTHlVLd/655sc+IfnuPmlj1haUZ10iWpPsnPgkCvhlNth0Qcw9gD48Nmkq5IkSWr1GhPa5wF9G7zuk97WGFtyrKRGyM/N5vS9tuGpyw7g2rN2p2dxPlc+OIURP3+K8256jX+/PpfllTVJl6n2YvDRMOY5KOoJt58AL/wB6h39IUmStLkac097DqmJ6A4mFbgnAqfHGN9bR9ubWfOe9q6kJp/bLd3kDVIT0S1Z3/t5T7u05d6dt4wH3/6Eh96az7ylK8nLyWLUwB4cPWwrDh7ck8I8R7SomVWtgAe/De/+G3Y8Eo7/O+R3SroqSZKkZpfUkm9fJbWkWzZwY4zxFyGEq4BJMcYHQgh7APcCXYBKYEGMcWj62K8DP0yf6hcxxps29F6GdqnpxBh5c85SHnzrEx5+ez6flVVRkJvNwYN7ctQuWzFqxx7k52YnXabaqhjh1WvhiR9B521SQ+d7DU26KkmSpGaVSGhvSYZ2qXnU1UcmzlrCQ29/wiPvLGBJeTUdO+Rw2JBeHDWslP126EFeTkZMc6G2ZvbL8K9zoKoMjv4L7PK1pCuSJElqNoZ2SVustq6el2cu5sG3PuGxdxewvLKWTgW5jB7am6OHbcXe23UlJ9sAryZUtgD+dR58/B/Y8yI47OeQk5d0VZIkSU3O0C6pSVXX1vPCBwt56O35PPHeAsqr6+jeMY8jdirlqF1K2aNfV7KyXP9dTaCuBp78CbzyN+izJ5x8C5RslXRVkiRJTcrQLqnZVNbU8dz0z3jwrfk8Pe1TKmvq6V2Sz1d3LuXoYaUM79uZEAzw2kLv3gP3XwJ5hXDSTdB//6QrkiRJajKGdkktoryqlqemfspDb8/n+ekLqa6rp0+XAo7cpZSjd9mKoVuVGOC1+T6bBuPOhCUz4dCfwlcuAT9PkiSpDdhYaA8h3AgcBXwWY9xpo+cztEvamOWVNTzx3qc89PYnvPjBImrrI/27F3H0LqUcNWwrBvYqTrpEtUaVy+H+b8DUB2HIsXDs36CDnyVJktS6NSK0jwRWALca2iU1uc/Lq3nsvQU8+NYnvDJzMfURBvTsyP4DerDfgG7s2b8bHTu4DrwaKUb4z1/gqSuh2w6pZeF67Jh0VZIkSZutMcPjQwj9gIcM7ZKa1WdllTz6zgKemLKAibM+p7q2npyswPC+ndl3h+7sN6A7w/t2JteZ6LUxH02Au78ONSvh2L/C0OOTrkiSJGmzhBCqgXcabBobYxy7Vpt+GNoltaTKmjpen/05L81YxEszFvH2vGXECEV52ezZv+vqEL9jr2Lvhde6Lf8Exp8Dc19L3eN+yE8h21EbkiSpdbGnXVKrsKyihpdnLl4d4mcuSv3/unvHPPbZvjv77dCdfXboRp8uhQlXqoxSWw1P/AheGwvb7gcn3QjFvZKuSpIkqdEM7ZJapU+Wrlwd4F/6cDELy6oA6NetkH136M6+O3TnK9t1o0tRXsKVKiO8NQ4evBTyO6XWc99m76QrkiRJahRDu6RWL8bIB5+t4MUPFvGfDxfxyswlrKiqJQTYaatO7LNDN/bboTt79OtKfm520uUqKQveTS0Lt2wOHP5L2HOMy8JJkqSM14jZ4/8JjAK6A58CP4kx3rDe9oZ2SUmrqavn7blLeWnGYl6csYg3P/6cmrpIXk4WI7btsronfuetO5GdZWhrV1YuhXsvhvcfhZ2/Bkf/GfI2+MW1JElSohrT075J5zO0S8o0FdW1vPbREl6asYgXZyxm6vzlAJTk57D3dt3Yf0B3Rg7swbbdDG/tQn09vPhHeObn0HNwalm4btsnXZUkSdI6GdoltTuLV1Txnw8Xp0P8IuZ+vhJI3Q9/wMAeHLBjD/berhuFec403qbNeBr+fQHU18Lx/4BBRyZdkSRJ0pcY2iW1e7MWlfP8+wt5/v2FvPzhYlbW1JGXncUe/bukQvzAngzs1dGl5dqipR/D+LPhkzdhv8vgoB9DlvMeSJKkzGFol6QGqmrrmDTr81SIn76Q6Z+WAdC7JJ+RA7tzwMCe7LdDdzoV5iZcqZpMTSU89gN4/WbYbhSceAMUdU+6KkmSJMDQLkkbNH/ZSia8v5AJ7y/ihQ8WsryylqwAu27ThZEDUkPpndCujXjjNnj4u1DUA06+FfrsnnRFkiRJhnZJaqzaunremruU56cv5PkPFvH23KXECF0Kc9l/QA9GDuzByIHd6Vmcn3Sp2lyfTIbxZ0HZAjjiN7D7eS4LJ0mSEmVol6TNtKS8mhc+SN0LP+H9RSxaUQXAkNISDtixBwcM7MFu23QhLycr4Uq1SSqWwD0XwoynYPgZcOQfILcg6aokSVI7ZWiXpCZQXx+ZumD56nvhX5/9ObX1kaK8bPbZoXt6Qrse9O1amHSpaoz6Onj+t/D8r6H3znDybdC1f9JVSZKkdsjQLknNoKyyhv98uJgJ7y/kuekLmbc0tazcdt2L2Gu7rgwuLWFwaQmDehdTnO+kdhnr/cdTve4EOOE6GHhY0hVJkqR2xtAuSc0sxsjMReWpe+HfX8jkOUtZtrJm9f6+XQsYkg7xg0tLGFJaQp8uBS4xlymWfATjzoJP34UDfpD6yfKWB0mS1DIM7ZLUwmKMzF9WydT5y9M/ZUydv5yPFpez6p/Q4g45DCotXh3kB5eWsGOvYgryXEM8EdUV8PBl8NY/YYdD4YSxUNg16aokSVI7YGiXpAxRUV3L9AVlq0P81PnLmbagjBVVtQBkBejXvWh1b/zgdKjvXZJvr3xLiBEm3QiP/gBKSuGU26F0WNJVSZKkNs7QLkkZrL4+MufzCqbOX86UBmF+7ucrV7fpUpjLoN4lDNlqVa98MTv07EiHHHvlm8XcSTD+bKhYnJpZftczk65IkiS1YYZ2SWqFllfWMK1BiJ86fznTPy2jsqYegJyswA49OzK4tIShW5UwrG9nhm5VQmFeTsKVtxHli+Du8+CjCbD7uXDEbyGnQ9JVSZKkNsjQLkltRF195KNF5WsE+Snzl/Pp8tT68VkBBvYqZuetO7FL384M69OJQb1LXEd+c9XVwrM/hxf/BFvtBiffCp37Jl2VJElqYwztktTGfVZWydtzlvH23KW8PW8Zb89dxpLyagDysrMYXFrMzn06sUufzgzr05kdenYkO8t75Btt6kNw339BVg6cdCNsf2DSFUmSpDbE0C5J7UyMkbmfr+TtuekgP3cZ78xbtnrCu8K8bHbaqlM6yHdiWJ/ObNut0MnuNmTRDBh3JiyaDgf+CPa7zGXhJElSkzC0S5Kor0+tJb8qxL89dynvfbKcqtrUPfKdCnLZpU+n1ND6Pp0Z1reTs9avrbocHvg2vHs37PhVOO7vUNA56aokSVIrZ2iXJK1TTV09739alg7xqSA/fUEZtfWpf+d7FHdgWJ9O7Lx1Z3bpm+qR71qUl3DVCYsRXr0WnvgRdN4GTr4Neu+UdFWSJKkVM7RLkhqtsqaOKfOX8/acL+6P/3DhClb909+nSwGDehczoFcxA3t1ZEDP1PJz+bntbPm5j1+B8edA5TI4+s8w7JSkK5IkSa2UoV2StEXKKmt4d95y3pmXGlr//qdlzFxYvrpHPivANl0LVwf5gb2KGdCzmO16FLXtMF/2aWpZuNkvwZ5j4LBfQE47H4kgSZI2maFdktTkaurqmbWonPc/XcH7n5bxwWdlvP/pCmYtWjPMb9utiAE900E+Hei361FEh5w2EubrauCpK+Hlv0KfPeHkW6Bkq6SrkiRJrYihXZLUYqpr6/loUfnqEP/Bp2W8/2kZsxZXUJcO89lZgW27FTKwZ3qIfTrQb9e9Y+tdU/69e+G+b0JeIZx0E/TfP+mKJElSK5FIaA8hjAb+DGQD18cYf73W/g7ArcDuwGLglBjjrBBCP2AqMD3d9JUY48Ubei9DuyRlvqraOj5K98yvCvIffLqCWYvLSWd5srMC/boVpnvlvxhqv03XwtYxzH7h9NSycIs/hEN+Avt8G5x9X5IkbUSLh/YQQjbwPnAoMBeYCJwWY5zSoM03gF1ijBeHEE4Fjo8xnpIO7Q/FGBs9Fa+hXZJar8qaOmYuTPXMf7B6qP0KZjcI8yHAVp0K6N+9iH7dC+nXrSj9vIi+XQozq3e+qgzu/yZMuR8GHwPHXQMdipOuSpIkZbAkQvtXgCtjjIenX18BEGP8VYM2j6fbvBxCyAEWAD2AbTG0S1K7V1lTx4cLVzDjsxV8tKicWYvK+WhxBR8tXMHyytrV7bKzAn26FHwR5LsV0q976vnWnQvIyU4g0MeYusf9yZ9At+3hlNuhx44tX4ckSWoVmjq05zSizdbAnAav5wJ7ra9NjLE2hLAM6Jbe1z+E8CawHPhxjPGFtd8ghDAGGAOQl+dMvZLU1uTnZjN0q04M3arTGttjjHxeUbM6yM9aXJ56vric12d/zoqqLwJ9bnagb9dC+ndL9cr3616Ufl7IVp0KyMpqpqHrIcA+34LS4anZ5cceCMf+FXY6oXneT5IkqYHGhPYtMR/YJsa4OISwO3BfCGFojHF5w0YxxrHAWEj1tDdzTZKkDBFCoGtRHl2L8th92y5r7IsxsnBFFbMWVaR75tM99IvKeenDRVTW1K9u2yEni227rTnUftXznsUdmibQ998fLpqQWs/97vNg3utwyJWQnbvl55YkSVqPxoT2eUDfBq/7pLetq83c9PD4TsDimBp7XwUQY3w9hPAhMBCYtKWFS5LathACPYvz6Vmcz579u66xr74+8mlZZbqHvmJ1D/1Hi8p57v2FVNd+EejzcrLo07mAPl0L6dulgL5dC+nbpZC+XQvo26WQzoW5hMZOMFeyFZz7MDzx49SQ+XlvwNduhuJeTfibS5IkfaEx97TnkJqI7mBS4XwicHqM8b0Gbb4J7NxgIroTYownhxB6AEtijHUhhO2AF9Ltlqzv/bynXZK0JerqI58sXcmsxeXMWlzB3CUVzPm8gjlLVjLn8wqWVtSs0b5jhxz6rCPM9+2ael6Yt57vt98eDw98G/I7pYL7tl9p/l9OkiRlvKSWfPsqcDWpJd9ujDH+IoRwFTApxvhACCEfuA3YFVgCnBpjnBlCOBG4CqgB6oGfxBgf3NB7GdolSc2prLJmdYCfs6SCuZ+vZE6DYL+ypm6N9t2K8tbbS7911Yfk/vscWPoxHPZz2Otil4WTJKmdSyS0tyRDuyQpKTFGFpdXp0P8ynSo/6KX/pOlK6mp++K/m1kBdiiu4xfhb+xR9QpTux3Gqzv9hOKSznQpyqVzYR5dCvPoUphLSX5u802WJ0mSMoahXZKkhNTVRxYsr/xSD/28xeWMWnQHY2rvZEbcmotrvsNHsXSNY7MCdCrIpUthHp0LVz2mAn2XoobbctNBP/U8Pzc7od9WkiRtDkO7JEkZqv6DZ+Ce86G2mln7/57ZPQ/m84pqPq+oYWlF9ZrPy1dtq/nSkPyGCnKz6VyY+0XAbxDse3fKp1962bvSTgVk25MvSVLiDO2SJGWypXNg/FnwyZuw73/DQf8D2RterKWypo6lFTXpUF+9+vnSiho+L/8i6C9d+cX2pRXV1Df4T3hedhZ9uxbQv3sR23Yrol+3wtVL323V2UAvSVJLMbRLkpTpairhsR/A6zdD/5Fw4o3QsUeTvkXDZe9mL04tezerwfOG69jnZgf6dkmF+FXr2acCfSFbdy4gJzurSWuTJKk9M7RLktRavHk7PHQZFHWHk2+FPiNa5G1jjHy6vIpZi8uZvbicjxZVMDu9BN7sxeVUVH8xHD8nK9C3a+EXYb5bIdume+j7dCkg10AvSdImMbRLktSafDI5NVx++Xw44jcw4uuJLgsXY2RhWRWzFlcwa1F5OthXpHvsyylvEOizswJ9uhSsHm7fragDHfNzKO6QQ8f8HDqmHxu+LsrLcZZ8SVK7ZmiXJKm1qVgC91wIM56CYafBkX+EvMKkq/qSGCOLVlR/aaj9rMXlzF5UQVlVbaPO07HDF4G+Y4ccilcF/LVCfnF+7jqDf3GHXIo6ZDtsX5LUKhnaJUlqjerr4fnfpH567QSn3AZd+ydd1SapraunvKqOsqoaVlTVsqKylrL045dfp9qUNdi3+rG6lsb8+dGxQ05qmbyiXDoX5NGpMJfOBbmrZ8/vVJCaVb9zentqfx55OYZ9SVJyDO2SJLVm7z8B91yQen7C9TDwsGTrSUB9faSipm51uC9bR+gvq6xl2coalq78Yrb8pStrWFZRw9KVNdTVr//vl6K8bDqvDvWpn04FqSXzOheu/QVAanunwlw65GS34FWQJLVVhnZJklq7JR+l7nNf8A4c8IPUT5aBsbFijKyoqk2H+QbBfmUNyypWLZFXw7IG21cF/9oNhP3SBuvep+7jTz/vWkRBnv/7SJIax9AuSVJbULMyNbP8W3fCDofACddBYdekq2rTYoyUV9eleu0ralI9+RU1fF5RzaIVVXy8pGL1vfyLy6vXOLZ3Sf6XlsvbNh3qC/NyEvqNJEmZyNAuSVJbESO8fhM8+gMo7g0n3wZbDU+6KgHLK2uYvahi9bJ5X8y2X8GiFVVrtO1Z3GHdPfTdiujYwUAvSe2NoV2SpLZm7usw/mwoXwhH/RF2PTPpirQBK6pq15xdv8Hzz8rWDPTdO3agX7fCNXrn+3cvorRTPvm52eTnZpPtEnmS1KYY2iVJaovKF8HdX4ePnofdzoEjfgu5+UlXpU1UXlXL7MUVa/XOp0L9guWV6zwmJyvQISeL/NxsOuRk0WHtx4b7crLJz009dsjdwL6cLDrkfrGvMC+HkvQye/m5WYTgFwWS1FwM7ZIktVX1dfDMz+HFP8JWu8LJt0LnbZKuSk2koro2fd98BZ8ur6Sqto7KmnqqauuoqqmnMv1YVVtPZU0dVbX1Ddp80a5h+5q6Tf87Licr0DE/h+L8HIo75KYe83Mpyc/5Ynt+7pqPHRpuy6EoL4csRwhI0joZ2iVJauumPQz3XgxZOXDSDbD9QUlXpAxVVx+pXmfIr1sd/itr6qmormV5ZS1llTWrl9Qrq6xJP9ayvLKGFVVfbN/AJPsAhAAdO3w5zK963rFDDoV5ORR1yKaoQw6Fedmrt3XskENhh2yK0vsL83K8RUBSm2JolySpPVj8IYw7Ez6bCgf9GPa7DLKykq5K7UCMkYrqutUBfnllbTrQ13wp7K/xuuqL7eVVtVTV1jf6PQtys1cH+KIOORTlpcL+qm0d08F/1b7CDl9s69ghh86FuXQqyKNTQS55Of7/RFKyDO2SJLUX1eXw4KXwzr9g4BFw/D+goHPSVUmNUlNXT0V1HRXVqRBfXlVHeXXqsSL9WF5Vm95WS3l1HRVVtaxYtb86tT+1LfW6bmNDAICivGw6F6YCfJeiXDoX5NGpMJfOBbl0LvzidZfCvPTrXDoV5tIhJ7sFroqk9sDQLklSexIjvDYWHv8hdOoLp9wOvXdKuiqpxcUYqa6rXx32K6rrUmG+qpZlK2tYurKGpeXVqceKGpatrGZpRc3q10srqqndQOgvyM1O99g3CPTpHvzOq0N/HiX5OeTlZJGbnUVOdiAvO/U8NyeL3KzwxfPsQG5Wlvf+S+2QoV2SpPbo41fhX+fAyqVw9J9h2ClJVyS1KjFGyqvrWFpRnQ716TC/KtxXfBHyl62xvYbqusYP9V9bdlZIBfhV4T79PC8d+ldt/9LrnEBOVmqFgIK8bArysinMzaEgL4uCvBwKc7MpzMsmPy87/TynQbvUY4ccVwqQkmBolySpvSr7NLUs3OwXYc8xcNgvICcv6aqkNi3GyMqautUBfnllDTV19dTWpXr+a1b/xNRjbep5dbrNqv0NX1eval9bT219PdXp5186V11q5YCK6jpW1tRRvQnzBABkhdQIgoK81P3/hXnZ5KfDfmFeantBbtbqwL8q7Bes2t/g2ILcL7YX5uaQn5f6osEvBaQvM7RLktSe1dXC01fCf/4P+uwJJ98CJVslXZWkFlBbV8/KmjpWpkN8as6Ahq9rWblqW7pd6vma29d1zMqaTV9CMDsrrBH0C3JXfRmQTUHuur8oyE+PCihscMyqx/wGzwtyUyMFvL1ArZGhXZIkwXv3wf3fhNwCOOkm6L9/0hVJauWqa1NfClSu/kKgdo0vCNZ8XrvG9jW/KKhlZU09K6trv9hf07iJBNe2+vaA3C8H+y+eZ6Ve52WTn7Nm8M9f49is1fMRrH1LQl52Frk5qec5WcERBNoihnZJkpSycHpqWbjFH8IhV8I+30otoC1JGai6tn6NHv6K6tQXBKvCfmVtPZWrwv+qbQ2er/pC4Yt99V+8Tm/blKUGN2TNeQiyGrxex/wEqyYmzArk5qS/AMgO5GRnkR0C2Vlr/YRAVvoxJzuQFQLZWZAVAjnpNllZqedZax/f4Njs7LDG+XOyAvnpLyc65GSvfp6fm01utkshtiRDuyRJ+kJVWarHfcr9MPgYOO4a6FCcdFWSlIj6+khl7Vohv7p+dbBfPXdA/frnEahOP6+t+2J+gnUfF790/NpzGdTVR2rrI/X1kboYqa+H2vp6NmPQwRb5cqDPSr9OP0+H/A6rRi2stT0/N4sOq7bnfPFFQIyR+giR1GN9jJB+jOnH1O/6xf7Y4DGSuib1MRLhi/PFtbbVR7btVsSBg3q27IXbTIZ2SZK0phjh5b/Bk/8Lu54Jx/wl6YokSRuwKpzW1cfUT4xfPK+P1K/1ui4dXGsb7F/9ZUB6f01dpKomPWKhpo6qmlVfXKReV9bUU1lbl96X3la75oiF1dvT59mcWxqay+FDe3HtWSOSLqNRDO2SJGndZv8Hug+Eou5JVyJJagNq6hoE/po6qtIhPzVxYT3ZIXX/f1aAEAIhpIb5B9KPgdXbslbvT7dlze2rj21wjobnzsvJomOHnISvSOMY2iVJkiRJylBNHdqdkUCSJEmSpAxlaJckSZIkKUMZ2iVJkiRJylCGdkmSJEmSMpShXZIkSZKkFhRCGB1CmB5CmBFCuHxDbQ3tkiRJkiS1kBBCNvA34AhgCHBaCGHI+tob2iVJkiRJajl7AjNijDNjjNXAXcCx62tsaJckSZIkqeVsDcxp8Hpuets6NSq0b2y8fQihQwhhXHr/qyGEfg32XZHePj2EcHjjfw9JkiRJklqdnBDCpAY/Y7boZBtr0GC8/aGkvgGYGEJ4IMY4pUGz84HPY4w7hBBOBX4DnJIel38qMBTYCngqhDAwxli3JUVLkiRJkpShamOMIzawfx7Qt8HrPult69SYnvbGjLc/Frgl/fxu4OAQQkhvvyvGWBVj/AiYkT6fJEmSJEnt0URgQAihfwghj1RH9wPra9yY0N6Y8far28QYa4FlQLdGHksIYcyqoQO1tbWNKEmSJEmSpNYnnZkvAR4HpgLjY4zvra/9RofHt4QY41hgLEBRUVFMuBxJkiRJkppNjPER4JHGtG1MT3tjxtuvbhNCyAE6AYsbeawkSZIkSVqHxoT2xoy3fwA4J/38JOCZGGNMbz81Pbt8f2AA8FrTlC5JkiRJUtu20eHxMcbaEMKq8fbZwI0xxvdCCFcBk2KMDwA3ALeFEGYAS0gFe9LtxgNTgFrgmxubOb6ioiKGEFZu4e/kjfEtw2vdsrzeLcdr3bK83i3Ha92yvN4tx2vdsrzeLcdr3bKa6noXNME5VgupDvG2I4QwaSPT66uJeK1blte75XitW5bXu+V4rVuW17vleK1blte75XitW1amXu/GDI+XJEmSJEkJMLRLkiRJkpSh2mJoH5t0Ae2I17pleb1bjte6ZXm9W47XumV5vVuO17pleb1bjte6ZWXk9W5z97RLkiRJktRWtMWedkmSJEmS2gRDuyRJkiRJGarNhPYQwugQwvQQwowQwuVJ19PWhRBmhRDeCSFMDiFMSrqetiaEcGMI4bMQwrsNtnUNITwZQvgg/dglyRrbivVc6ytDCPPSn+/JIYSvJlljWxFC6BtCeDaEMCWE8F4I4dL0dj/bzWAD19vPdxMLIeSHEF4LIbyVvtY/TW/vH0J4Nf23ybgQQl7StbYFG7jeN4cQPmrw2R6ecKltRgghO4TwZgjhofRrP9vNaB3X2892M1lXpsnEv0vaRGgPIWQDfwOOAIYAp4UQhiRbVbtwYIxxeCauZdgG3AyMXmvb5cDTMcYBwNPp19pyN/Plaw3wp/Tne3iM8ZEWrqmtqgW+G2McAuwNfDP9b7Wf7eaxvusNfr6bWhVwUIxxGDAcGB1C2Bv4DalrvQPwOXB+ciW2Keu73gDfb/DZnpxUgW3QpcDUBq/9bDevta83+NluTmtnmoz7u6RNhHZgT2BGjHFmjLEauAs4NuGapM0WY5wALFlr87HALenntwDHtWRNbdV6rrWaQYxxfozxjfTzMlJ/kGyNn+1msYHrrSYWU1akX+amfyJwEHB3eruf7SaygeutZhBC6AMcCVyffh3ws91s1r7eSkTG/V3SVkL71sCcBq/n4h8mzS0CT4QQXg8hjEm6mHaiV4xxfvr5AqBXksW0A5eEEN5OD59PfFhUWxNC6AfsCryKn+1mt9b1Bj/fTS49nHUy8BnwJPAhsDTGWJtu4t8mTWjt6x1jXPXZ/kX6s/2nEEKH5CpsU64G/h9Qn37dDT/bzelq1rzeq/jZbh7ryjQZ93dJWwntann7xRh3I3VLwjdDCCOTLqg9iam1Gu1VaD5/B7YnNexyPvCHRKtpY0IIHYF/A/8dY1zecJ+f7aa3juvt57sZxBjrYozDgT6kRgAOSraitm3t6x1C2Am4gtR13wPoCvwguQrbhhDCUcBnMcbXk66lPdjA9faz3Xw2mGky5e+SthLa5wF9G7zuk96mZhJjnJd+/Ay4l9QfKGpen4YQSgHSj58lXE+bFWP8NP0HYT1wHX6+m0wIIZdUgLwjxnhPerOf7Wayruvt57t5xRiXAs8CXwE6hxBy0rv826QZNLjeo9O3hMQYYxVwE362m8K+wDEhhFmkbj89CPgzfraby5eudwjhdj/bzWc9mSbj/i5pK6F9IjAgPZNlHnAq8EDCNbVZIYSiEELxqufAYcC7Gz5KTeAB4Jz083OA+xOspU1b9Q912vH4+W4S6fsgbwCmxhj/2GCXn+1msL7r7ee76YUQeoQQOqefFwCHkppD4FngpHQzP9tNZD3Xe1qDP7IDqXtQ/WxvoRjjFTHGPjHGfqT+vn4mxngGfrabxXqu95l+tpvHBjJNxv1dkrPxJpkvxlgbQrgEeBzIBm6MMb6XcFltWS/g3tS/G+QAd8YYH0u2pLYlhPBPYBTQPYQwF/gJ8GtgfAjhfGA2cHJyFbYd67nWo9LLqURgFnBRUvW1MfsCZwHvpO9FBfghfraby/qu92l+vptcKXBLejWbLGB8jPGhEMIU4K4Qws+BN0l9iaItt77r/UwIoQcQgMnAxQnW2Nb9AD/bLekOP9vNYp2ZJoQwkQz7uySkhulLkiRJkqRM01aGx0uSJEmS1OYY2iVJkiRJylCGdkmSJEmSMpShXZIkSZKkDGVolyRJkiQpQxnaJUnSGkIIo0IIDyVdhyRJMrRLkiRJkpSxDO2SJLVSIYQzQwivhRAmhxCuDSFkhxBWhBD+FEJ4L4TwdAihR7rt8BDCKyGEt0MI94YQuqS37xBCeCqE8FYI4Y0Qwvbp03cMIdwdQpgWQrgjhBAS+0UlSWrHDO2SJLVCIYTBwCnAvjHG4UAdcAZQBEyKMQ4Fngd+kj7kVuAHMcZdgHcabL8D+FuMcRiwDzA/vX1X4L+BIcB2wL7N/CtJkqR1yEm6AEmStFkOBnYHJqY7wQuAz4B6YFy6ze3APSGETkDnGOPz6e23AP8KIRQDW8cY7wWIMVYCpM/3Woxxbvr1ZKAf8GKz/1aSJGkNhnZJklqnANwSY7xijY0h/M9a7eJmnr+qwfM6/JtBkqREODxekqTW6WngpBBCT4AQQtcQwrak/tt+UrrN6cCLMcZlwOchhP3T288Cno8xlgFzQwjHpc/RIYRQ2JK/hCRJ2jC/NZckqRWKMU4JIfwYeCKEkAXUAN8EyoE90/s+I3XfO8A5wD/SoXwmcF56+1nAtSGEq9Ln+FoL/hqSJGkjQoybO2pOkiRlmhDCihhjx6TrkCRJTcPh8ZIkSZIkZSh72iVJkiRJylD2tEuSJEmSlKEM7ZIkSZIkZShDuyRJkiRJGcrQLkmSJElShjK0S5IkSZKUof4/liarq6NwnpUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1224x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(predfolder)\n",
    "print(state['log_history'].iloc[-1])\n",
    "epwiseloss.plot(x='epoch',y=['loss','learning_rate'],kind='line',secondary_y='learning_rate',figsize=(17,7))\n",
    "plt.xticks(np.arange(0,101,5))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "supreme-victoria",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "accepted-associate",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8     0.0781\n",
       "9     0.0694\n",
       "10    0.0637\n",
       "11    0.0573\n",
       "12    0.0521\n",
       "13    0.0476\n",
       "14    0.0436\n",
       "15    0.0413\n",
       "16    0.0369\n",
       "17    0.0352\n",
       "18    0.0330\n",
       "19    0.0286\n",
       "20    0.0276\n",
       "21    0.0258\n",
       "22    0.0243\n",
       "23    0.0221\n",
       "24    0.0214\n",
       "25    0.0201\n",
       "26    0.0188\n",
       "27    0.0176\n",
       "28    0.0169\n",
       "29    0.0165\n",
       "30    0.0154\n",
       "31    0.0156\n",
       "32       NaN\n",
       "Name: loss, dtype: float64"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epwiseloss['loss'].iloc[-25:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "secret-corruption",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "everyday-aurora",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}