Embedding-RNN-Autoencoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.501032Z",
     "start_time": "2017-05-08T09:40:33.359980Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch import nn, ones\n",
    "from torch.autograd import Variable\n",
    "from torchvision import models\n",
    "from torch.nn.init import kaiming_normal\n",
    "from torch import np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import random\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playing with embeddings - simple classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-07T15:25:27.962041Z",
     "start_time": "2017-05-07T15:25:27.959472Z"
    }
   },
   "source": [
    "## Encoding functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.507314Z",
     "start_time": "2017-05-08T09:40:33.502508Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<BEGIN>': 0, '<STOP>': 1, 'clear': 2, 'cloudy': 3, 'haze': 4, 'partly_cloudy': 5, 'agriculture': 6, 'artisinal_mine': 7, 'bare_ground': 8, 'blooming': 9, 'blow_down': 10, 'conventional_mine': 11, 'cultivation': 12, 'habitation': 13, 'primary': 14, 'road': 15, 'selective_logging': 16, 'slash_burn': 17, 'water': 18}\n"
     ]
    }
   ],
   "source": [
    "vocab = ['<BEGIN>','<STOP>','clear', 'cloudy', 'haze','partly_cloudy',\n",
    "    'agriculture','artisinal_mine','bare_ground','blooming',\n",
    "    'blow_down','conventional_mine','cultivation','habitation',\n",
    "    'primary','road','selective_logging','slash_burn','water'\n",
    "    ]\n",
    "\n",
    "word_to_ix = { word: i for i, word in enumerate(vocab) }\n",
    "print(word_to_ix)\n",
    "one_hot_mapping = {k:np.eye(19)[v] for k,v in word_to_ix.items()}\n",
    "# print(one_hot_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.514648Z",
     "start_time": "2017-05-08T09:40:33.508295Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def label_to_target(word_to_ix, label):\n",
    "    return Variable(torch.LongTensor(\n",
    "            list(map(lambda w: word_to_ix[w], label))\n",
    "        ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-07T14:20:34.444983Z",
     "start_time": "2017-05-07T14:20:34.441966Z"
    }
   },
   "source": [
    "## Decoding functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.519410Z",
     "start_time": "2017-05-08T09:40:33.515671Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ix_to_word = {v: k for k, v in word_to_ix.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.528957Z",
     "start_time": "2017-05-08T09:40:33.520429Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "\n",
    "def predictions_to_label(ix_to_word, predictions):\n",
    "    predictions = F.softmax(predictions)\n",
    "    _, preds = torch.max(predictions.data, 1)\n",
    "    return list(map(lambda ix: ix_to_word[ix], flatten(preds.tolist())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.534073Z",
     "start_time": "2017-05-08T09:40:33.530224Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_batch(n, vocab):\n",
    "    batch = []\n",
    "    for _ in range(n):\n",
    "        batch.append(random.choice(vocab))\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.546459Z",
     "start_time": "2017-05-08T09:40:33.535248Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class EmbeddingIdentity(nn.Module):\n",
    "    \"\"\" Testing weight sharing\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, vocab, repr_dim, num_rnn_layers):\n",
    "        super(EmbeddingIdentity, self).__init__()\n",
    "        self.label_to_ix = { label: i for i, label in enumerate(vocab) }\n",
    "        self.embeds = nn.Embedding(len(vocab), repr_dim)\n",
    "        self.rnn = nn.LSTM(input_size=repr_dim,\n",
    "                            hidden_size=repr_dim,\n",
    "                            num_layers=num_rnn_layers,\n",
    "                            batch_first = True)\n",
    "        self.fc = nn.Linear(repr_dim, len(vocab))\n",
    "        \n",
    "        # link embedding and decoding weight\n",
    "        self.fc.weight = self.embeds.weight\n",
    "    \n",
    "    def toVariable(self, x):\n",
    "        return Variable(torch.LongTensor(\n",
    "            list(map(lambda lbl: self.label_to_ix[lbl], x))\n",
    "        ))\n",
    "    \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.toVariable(x)          # Tensor with a single integer\n",
    "        f = self.embeds(x).unsqueeze(1) # Dim 1x5 --> unsqueeze --> 1x1x5\n",
    "        f, hidden = self.rnn(f, hidden) # Dim output: 1x1x5, Dim hidden: 2x1x5\n",
    "        f = self.fc(f.contiguous().squeeze(1)) # Dim 1x19\n",
    "        return f\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.558006Z",
     "start_time": "2017-05-08T09:40:33.547922Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = EmbeddingIdentity(vocab,5, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.571745Z",
     "start_time": "2017-05-08T09:40:33.559124Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['road']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = model([\"slash_burn\"])\n",
    "predictions_to_label(ix_to_word, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.577884Z",
     "start_time": "2017-05-08T09:40:33.575433Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch = gen_batch(10, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.590066Z",
     "start_time": "2017-05-08T09:40:33.579082Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bare_ground',\n",
       " 'bare_ground',\n",
       " 'road',\n",
       " 'bare_ground',\n",
       " 'road',\n",
       " 'road',\n",
       " 'bare_ground',\n",
       " 'road',\n",
       " 'bare_ground',\n",
       " 'bare_ground']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = model(batch)\n",
    "predictions_to_label(ix_to_word, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.598216Z",
     "start_time": "2017-05-08T09:40:33.593700Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['road',\n",
       " 'selective_logging',\n",
       " 'slash_burn',\n",
       " 'partly_cloudy',\n",
       " 'water',\n",
       " 'slash_burn',\n",
       " 'blow_down',\n",
       " 'blooming',\n",
       " 'clear',\n",
       " 'cultivation']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = label_to_target(word_to_ix,batch)\n",
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.606387Z",
     "start_time": "2017-05-08T09:40:33.600299Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = F.cross_entropy(output, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.616958Z",
     "start_time": "2017-05-08T09:40:33.609928Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 3.1522\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loss.backward()\n",
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wrapping up for multi epoch training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:33.621777Z",
     "start_time": "2017-05-08T09:40:33.618931Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.214160Z",
     "start_time": "2017-05-08T09:40:33.623162Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 3.0327892303466797\n",
      "1 3.008108377456665\n",
      "2 3.017551898956299\n",
      "3 3.005432367324829\n",
      "4 2.9789912700653076\n",
      "5 2.9626643657684326\n",
      "6 2.976961135864258\n",
      "7 2.970008611679077\n",
      "8 2.9364094734191895\n",
      "9 2.941561222076416\n",
      "10 2.9203896522521973\n",
      "11 2.9210522174835205\n",
      "12 2.909860849380493\n",
      "13 2.8913323879241943\n",
      "14 2.866137981414795\n",
      "15 2.8700342178344727\n",
      "16 2.8909213542938232\n",
      "17 2.8746190071105957\n",
      "18 2.836191177368164\n",
      "19 2.8244004249572754\n",
      "20 2.824340343475342\n",
      "21 2.794071912765503\n",
      "22 2.7842977046966553\n",
      "23 2.7581307888031006\n",
      "24 2.762523889541626\n",
      "25 2.741461753845215\n",
      "26 2.6938722133636475\n",
      "27 2.6771738529205322\n",
      "28 2.6585323810577393\n",
      "29 2.642106056213379\n",
      "30 2.622920513153076\n",
      "31 2.574413299560547\n",
      "32 2.5437347888946533\n",
      "33 2.529202938079834\n",
      "34 2.5020318031311035\n",
      "35 2.4690897464752197\n",
      "36 2.4492805004119873\n",
      "37 2.360297441482544\n",
      "38 2.376901865005493\n",
      "39 2.321429967880249\n",
      "40 2.263556718826294\n",
      "41 2.2434051036834717\n",
      "42 2.1823229789733887\n",
      "43 2.1596827507019043\n",
      "44 2.1252403259277344\n",
      "45 2.0954039096832275\n",
      "46 2.069568157196045\n",
      "47 2.0060946941375732\n",
      "48 1.968686819076538\n",
      "49 1.9379212856292725\n",
      "50 1.8821163177490234\n",
      "51 1.8651610612869263\n",
      "52 1.8083884716033936\n",
      "53 1.7988858222961426\n",
      "54 1.7624099254608154\n",
      "55 1.719488263130188\n",
      "56 1.6745964288711548\n",
      "57 1.664591908454895\n",
      "58 1.6227556467056274\n",
      "59 1.5678842067718506\n",
      "60 1.5629092454910278\n",
      "61 1.5608023405075073\n",
      "62 1.479115605354309\n",
      "63 1.5049073696136475\n",
      "64 1.4646766185760498\n",
      "65 1.418389916419983\n",
      "66 1.3770095109939575\n",
      "67 1.3775206804275513\n",
      "68 1.3475971221923828\n",
      "69 1.3333832025527954\n",
      "70 1.2985622882843018\n",
      "71 1.2942696809768677\n",
      "72 1.2746336460113525\n",
      "73 1.2487176656723022\n",
      "74 1.2092317342758179\n",
      "75 1.2098724842071533\n",
      "76 1.194258451461792\n",
      "77 1.1698545217514038\n",
      "78 1.1748318672180176\n",
      "79 1.1295673847198486\n",
      "80 1.1253141164779663\n",
      "81 1.1195603609085083\n",
      "82 1.0939793586730957\n",
      "83 1.0856468677520752\n",
      "84 1.0619988441467285\n",
      "85 1.0312669277191162\n",
      "86 1.0308483839035034\n",
      "87 1.0175026655197144\n",
      "88 1.0337523221969604\n",
      "89 0.9955971240997314\n",
      "90 1.004127860069275\n",
      "91 0.9742776155471802\n",
      "92 0.9346387982368469\n",
      "93 0.9524136185646057\n",
      "94 0.9387719035148621\n",
      "95 0.9159666895866394\n",
      "96 0.9207743406295776\n",
      "97 0.8729625344276428\n",
      "98 0.8788477778434753\n",
      "99 0.8707550764083862\n",
      "100 0.8460355401039124\n",
      "101 0.8683237433433533\n",
      "102 0.8368710279464722\n",
      "103 0.8485000133514404\n",
      "104 0.823924720287323\n",
      "105 0.8203246593475342\n",
      "106 0.8045764565467834\n",
      "107 0.7839118838310242\n",
      "108 0.7617170810699463\n",
      "109 0.7477543354034424\n",
      "110 0.7654331922531128\n",
      "111 0.7484622597694397\n",
      "112 0.739612877368927\n",
      "113 0.7274085283279419\n",
      "114 0.7198834419250488\n",
      "115 0.7208585739135742\n",
      "116 0.6990858912467957\n",
      "117 0.6606507301330566\n",
      "118 0.680462121963501\n",
      "119 0.6707110404968262\n",
      "120 0.6412085890769958\n",
      "121 0.6421482563018799\n",
      "122 0.6433846950531006\n",
      "123 0.6438480615615845\n",
      "124 0.6214024424552917\n",
      "125 0.6137454509735107\n",
      "126 0.6015021204948425\n",
      "127 0.5954461693763733\n",
      "128 0.6114381551742554\n",
      "129 0.590767502784729\n",
      "130 0.5792084336280823\n",
      "131 0.5702701807022095\n",
      "132 0.5781854391098022\n",
      "133 0.5436075329780579\n",
      "134 0.5652246475219727\n",
      "135 0.5529614686965942\n",
      "136 0.5338315963745117\n",
      "137 0.519795835018158\n",
      "138 0.5103486180305481\n",
      "139 0.5065822601318359\n",
      "140 0.4920341670513153\n",
      "141 0.4955759048461914\n",
      "142 0.49055173993110657\n",
      "143 0.4862269461154938\n",
      "144 0.4892207980155945\n",
      "145 0.4582788944244385\n",
      "146 0.47219282388687134\n",
      "147 0.45616376399993896\n",
      "148 0.4476357698440552\n",
      "149 0.4435167908668518\n",
      "150 0.42782536149024963\n",
      "151 0.4359758794307709\n",
      "152 0.43610885739326477\n",
      "153 0.4222642183303833\n",
      "154 0.4205371141433716\n",
      "155 0.39887285232543945\n",
      "156 0.421176552772522\n",
      "157 0.3988403081893921\n",
      "158 0.38862431049346924\n",
      "159 0.38794952630996704\n",
      "160 0.3744138479232788\n",
      "161 0.39346495270729065\n",
      "162 0.3817007541656494\n",
      "163 0.3639132082462311\n",
      "164 0.36059844493865967\n",
      "165 0.360429972410202\n",
      "166 0.36135169863700867\n",
      "167 0.34890633821487427\n",
      "168 0.3453003168106079\n",
      "169 0.33418503403663635\n",
      "170 0.34763142466545105\n",
      "171 0.3324037194252014\n",
      "172 0.32130685448646545\n",
      "173 0.3212946057319641\n",
      "174 0.3212886452674866\n",
      "175 0.3216099441051483\n",
      "176 0.31211861968040466\n",
      "177 0.3148205578327179\n",
      "178 0.29900264739990234\n",
      "179 0.3048059344291687\n",
      "180 0.2956213057041168\n",
      "181 0.29743194580078125\n",
      "182 0.30284515023231506\n",
      "183 0.28489625453948975\n",
      "184 0.2780754566192627\n",
      "185 0.28084757924079895\n",
      "186 0.26822924613952637\n",
      "187 0.27760809659957886\n",
      "188 0.2688874304294586\n",
      "189 0.27149075269699097\n",
      "190 0.2661140263080597\n",
      "191 0.26533111929893494\n",
      "192 0.25197863578796387\n",
      "193 0.2583406865596771\n",
      "194 0.2555263638496399\n",
      "195 0.24990972876548767\n",
      "196 0.24757567048072815\n",
      "197 0.24341966211795807\n",
      "198 0.2450764924287796\n",
      "199 0.2447497695684433\n",
      "200 0.23898813128471375\n",
      "201 0.2391396015882492\n",
      "202 0.23360632359981537\n",
      "203 0.23138028383255005\n",
      "204 0.22629953920841217\n",
      "205 0.2267197072505951\n",
      "206 0.21933725476264954\n",
      "207 0.22016118466854095\n",
      "208 0.21508437395095825\n",
      "209 0.21666623651981354\n",
      "210 0.21921880543231964\n",
      "211 0.210036963224411\n",
      "212 0.20586630702018738\n",
      "213 0.2034505158662796\n",
      "214 0.2024817317724228\n",
      "215 0.1923520267009735\n",
      "216 0.19824443757534027\n",
      "217 0.19988001883029938\n",
      "218 0.1934642344713211\n",
      "219 0.19550968706607819\n",
      "220 0.18182379007339478\n",
      "221 0.18803821504116058\n",
      "222 0.1859552413225174\n",
      "223 0.17897896468639374\n",
      "224 0.19066305458545685\n",
      "225 0.1811107099056244\n",
      "226 0.17551907896995544\n",
      "227 0.16896113753318787\n",
      "228 0.17340272665023804\n",
      "229 0.17511484026908875\n",
      "230 0.17783688008785248\n",
      "231 0.1650107502937317\n",
      "232 0.16443714499473572\n",
      "233 0.1652948558330536\n",
      "234 0.16669252514839172\n",
      "235 0.1607659012079239\n",
      "236 0.16076259315013885\n",
      "237 0.16079089045524597\n",
      "238 0.15714655816555023\n",
      "239 0.1577678620815277\n",
      "240 0.1571982353925705\n",
      "241 0.1530398428440094\n",
      "242 0.15522348880767822\n",
      "243 0.15289755165576935\n",
      "244 0.15239793062210083\n",
      "245 0.14028388261795044\n",
      "246 0.15065713226795197\n",
      "247 0.14545032382011414\n",
      "248 0.14153267443180084\n",
      "249 0.14096444845199585\n",
      "250 0.14007584750652313\n",
      "251 0.14427050948143005\n",
      "252 0.13623279333114624\n",
      "253 0.13817670941352844\n",
      "254 0.13990925252437592\n",
      "255 0.13654859364032745\n",
      "256 0.13333432376384735\n",
      "257 0.1328703910112381\n",
      "258 0.13052566349506378\n",
      "259 0.13165926933288574\n",
      "260 0.12921380996704102\n",
      "261 0.12858670949935913\n",
      "262 0.12344743311405182\n",
      "263 0.11986885219812393\n",
      "264 0.1256721019744873\n",
      "265 0.12394973635673523\n",
      "266 0.12307633459568024\n",
      "267 0.12598635256290436\n",
      "268 0.123048335313797\n",
      "269 0.12593798339366913\n",
      "270 0.12365783751010895\n",
      "271 0.1235833540558815\n",
      "272 0.11443904787302017\n",
      "273 0.11731912940740585\n",
      "274 0.11431959271430969\n",
      "275 0.1121121495962143\n",
      "276 0.11599203199148178\n",
      "277 0.11310208588838577\n",
      "278 0.11197537183761597\n",
      "279 0.11446888744831085\n",
      "280 0.11789549887180328\n",
      "281 0.11309249699115753\n",
      "282 0.11117762327194214\n",
      "283 0.11133625358343124\n",
      "284 0.11165276914834976\n",
      "285 0.10946659743785858\n",
      "286 0.10790839791297913\n",
      "287 0.10328122228384018\n",
      "288 0.10567207634449005\n",
      "289 0.10699284076690674\n",
      "290 0.09533876925706863\n",
      "291 0.11170291900634766\n",
      "292 0.10740119218826294\n",
      "293 0.1042068749666214\n",
      "294 0.1041877344250679\n",
      "295 0.10094548761844635\n",
      "296 0.09660549461841583\n",
      "297 0.10359460115432739\n",
      "298 0.1000596433877945\n",
      "299 0.10058246552944183\n",
      "300 0.099457748234272\n",
      "301 0.0942087471485138\n",
      "302 0.09173782914876938\n",
      "303 0.0928456261754036\n",
      "304 0.09718280285596848\n",
      "305 0.09551634639501572\n",
      "306 0.0921732485294342\n",
      "307 0.09125163406133652\n",
      "308 0.09208497405052185\n",
      "309 0.09391967207193375\n",
      "310 0.09057430177927017\n",
      "311 0.09000656008720398\n",
      "312 0.08846519887447357\n",
      "313 0.08660370111465454\n",
      "314 0.09112387150526047\n",
      "315 0.08975087106227875\n",
      "316 0.08706732839345932\n",
      "317 0.08868413418531418\n",
      "318 0.08601889759302139\n",
      "319 0.0870109349489212\n",
      "320 0.08586764335632324\n",
      "321 0.08369702845811844\n",
      "322 0.08688191324472427\n",
      "323 0.08590886741876602\n",
      "324 0.08722055703401566\n",
      "325 0.08106348663568497\n",
      "326 0.08002591878175735\n",
      "327 0.08348486572504044\n",
      "328 0.08306070417165756\n",
      "329 0.08353058993816376\n",
      "330 0.08127650618553162\n",
      "331 0.08289020508527756\n",
      "332 0.0782008096575737\n",
      "333 0.07746326178312302\n",
      "334 0.08262383937835693\n",
      "335 0.07681377232074738\n",
      "336 0.08204783499240875\n",
      "337 0.08354604989290237\n",
      "338 0.07870259135961533\n",
      "339 0.07746938616037369\n",
      "340 0.077939972281456\n",
      "341 0.07307103276252747\n",
      "342 0.0790318176150322\n",
      "343 0.07614941149950027\n",
      "344 0.07318364083766937\n",
      "345 0.07589908689260483\n",
      "346 0.07729699462652206\n",
      "347 0.07277067005634308\n",
      "348 0.07280939072370529\n",
      "349 0.07087063044309616\n",
      "350 0.07130294293165207\n",
      "351 0.07075147330760956\n",
      "352 0.06937259435653687\n",
      "353 0.06867659091949463\n",
      "354 0.06847412139177322\n",
      "355 0.06786567717790604\n",
      "356 0.07145173102617264\n",
      "357 0.07056700438261032\n",
      "358 0.06724002212285995\n",
      "359 0.06906706094741821\n",
      "360 0.06702205538749695\n",
      "361 0.06591663509607315\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "362 0.06461622565984726\n",
      "363 0.06301934272050858\n",
      "364 0.06491687148809433\n",
      "365 0.0638754591345787\n",
      "366 0.06364478915929794\n",
      "367 0.06462763249874115\n",
      "368 0.06405320018529892\n",
      "369 0.06570523977279663\n",
      "370 0.0616951659321785\n",
      "371 0.06168324500322342\n",
      "372 0.061845507472753525\n",
      "373 0.05868249386548996\n",
      "374 0.06011258065700531\n",
      "375 0.06058717891573906\n",
      "376 0.05790421739220619\n",
      "377 0.06199761480093002\n",
      "378 0.059761274605989456\n",
      "379 0.0572650283575058\n",
      "380 0.056322094053030014\n",
      "381 0.05799992382526398\n",
      "382 0.056367985904216766\n",
      "383 0.05416860431432724\n",
      "384 0.05654078349471092\n",
      "385 0.05512376129627228\n",
      "386 0.05502678453922272\n",
      "387 0.0556374192237854\n",
      "388 0.05215751752257347\n",
      "389 0.05562286823987961\n",
      "390 0.05423089489340782\n",
      "391 0.05449938029050827\n",
      "392 0.05499233305454254\n",
      "393 0.051911719143390656\n",
      "394 0.05303974449634552\n",
      "395 0.051560044288635254\n",
      "396 0.0517883338034153\n",
      "397 0.050359077751636505\n",
      "398 0.04952573776245117\n",
      "399 0.05212709680199623\n",
      "400 0.04838406667113304\n",
      "401 0.04988162964582443\n",
      "402 0.050857122987508774\n",
      "403 0.050419602543115616\n",
      "404 0.05004628002643585\n",
      "405 0.048867061734199524\n",
      "406 0.04716965928673744\n",
      "407 0.046670202165842056\n",
      "408 0.046373460441827774\n",
      "409 0.04820006340742111\n",
      "410 0.04562713950872421\n",
      "411 0.04687552899122238\n",
      "412 0.04589962214231491\n",
      "413 0.04734596237540245\n",
      "414 0.045841194689273834\n",
      "415 0.04568338394165039\n",
      "416 0.04599996283650398\n",
      "417 0.04522385448217392\n",
      "418 0.04443953186273575\n",
      "419 0.04468422755599022\n",
      "420 0.04423638433218002\n",
      "421 0.04461774602532387\n",
      "422 0.043096501380205154\n",
      "423 0.043161213397979736\n",
      "424 0.04378198832273483\n",
      "425 0.043327365070581436\n",
      "426 0.04184993728995323\n",
      "427 0.04198940098285675\n",
      "428 0.04180886223912239\n",
      "429 0.04030512273311615\n",
      "430 0.04339122399687767\n",
      "431 0.04136202111840248\n",
      "432 0.04066752269864082\n",
      "433 0.042345087975263596\n",
      "434 0.04041742905974388\n",
      "435 0.0395362451672554\n",
      "436 0.04114943742752075\n",
      "437 0.04111358895897865\n",
      "438 0.04078586399555206\n",
      "439 0.04051497206091881\n",
      "440 0.0392443984746933\n",
      "441 0.040330179035663605\n",
      "442 0.03996222838759422\n",
      "443 0.04065973311662674\n",
      "444 0.03898729011416435\n",
      "445 0.037589181214571\n",
      "446 0.0377788245677948\n",
      "447 0.037823036313056946\n",
      "448 0.038363996893167496\n",
      "449 0.037884972989559174\n",
      "450 0.03579403832554817\n",
      "451 0.03754584118723869\n",
      "452 0.03774760290980339\n",
      "453 0.03728939965367317\n",
      "454 0.03553058207035065\n",
      "455 0.03595805540680885\n",
      "456 0.03553782403469086\n",
      "457 0.036989688873291016\n",
      "458 0.035599883645772934\n",
      "459 0.03595425933599472\n",
      "460 0.035077888518571854\n",
      "461 0.035797104239463806\n",
      "462 0.03685653209686279\n",
      "463 0.03596925735473633\n",
      "464 0.035600822418928146\n",
      "465 0.0347730815410614\n",
      "466 0.035101182758808136\n",
      "467 0.034784138202667236\n",
      "468 0.03429194912314415\n",
      "469 0.03384147584438324\n",
      "470 0.032767605036497116\n",
      "471 0.0338020883500576\n",
      "472 0.034400440752506256\n",
      "473 0.033890776336193085\n",
      "474 0.03298873454332352\n",
      "475 0.033635273575782776\n",
      "476 0.033850524574518204\n",
      "477 0.03226730599999428\n",
      "478 0.0334080308675766\n",
      "479 0.03190720081329346\n",
      "480 0.03230598196387291\n",
      "481 0.031858719885349274\n",
      "482 0.032274793833494186\n",
      "483 0.031114086508750916\n",
      "484 0.031922053545713425\n",
      "485 0.03132786601781845\n",
      "486 0.03117886371910572\n",
      "487 0.03083803318440914\n",
      "488 0.03133169561624527\n",
      "489 0.031130919232964516\n",
      "490 0.03073568269610405\n",
      "491 0.030889617279171944\n",
      "492 0.0299163069576025\n",
      "493 0.030124176293611526\n",
      "494 0.03004493936896324\n",
      "495 0.030596556141972542\n",
      "496 0.0304100438952446\n",
      "497 0.030152922496199608\n",
      "498 0.030491245910525322\n",
      "499 0.029634026810526848\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(500):\n",
    "    X = gen_batch(500, vocab)\n",
    "    y = label_to_target(word_to_ix,X)\n",
    "    y_pred = model(X)\n",
    "    loss = criterion(y_pred, y)\n",
    "    print(epoch, loss.data[0])\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.218317Z",
     "start_time": "2017-05-08T09:40:36.216021Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch2 = gen_batch(30, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.226091Z",
     "start_time": "2017-05-08T09:40:36.221088Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output2 = model(batch2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.231495Z",
     "start_time": "2017-05-08T09:40:36.227443Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['clear',\n",
       " 'habitation',\n",
       " 'conventional_mine',\n",
       " 'slash_burn',\n",
       " 'blooming',\n",
       " 'water',\n",
       " 'road',\n",
       " 'conventional_mine',\n",
       " 'partly_cloudy',\n",
       " '<BEGIN>',\n",
       " 'artisinal_mine',\n",
       " 'habitation',\n",
       " 'cloudy',\n",
       " 'selective_logging',\n",
       " 'artisinal_mine',\n",
       " 'conventional_mine',\n",
       " 'slash_burn',\n",
       " 'agriculture',\n",
       " 'artisinal_mine',\n",
       " 'conventional_mine',\n",
       " 'habitation',\n",
       " 'artisinal_mine',\n",
       " 'blow_down',\n",
       " 'agriculture',\n",
       " 'blow_down',\n",
       " 'water',\n",
       " 'conventional_mine',\n",
       " 'habitation',\n",
       " 'selective_logging',\n",
       " 'primary']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.236942Z",
     "start_time": "2017-05-08T09:40:36.232834Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['clear',\n",
       " 'habitation',\n",
       " 'conventional_mine',\n",
       " 'slash_burn',\n",
       " 'blooming',\n",
       " 'water',\n",
       " 'road',\n",
       " 'conventional_mine',\n",
       " 'partly_cloudy',\n",
       " '<BEGIN>',\n",
       " 'artisinal_mine',\n",
       " 'habitation',\n",
       " 'cloudy',\n",
       " 'selective_logging',\n",
       " 'artisinal_mine',\n",
       " 'conventional_mine',\n",
       " 'slash_burn',\n",
       " 'agriculture',\n",
       " 'artisinal_mine',\n",
       " 'conventional_mine',\n",
       " 'habitation',\n",
       " 'artisinal_mine',\n",
       " 'blow_down',\n",
       " 'agriculture',\n",
       " 'blow_down',\n",
       " 'water',\n",
       " 'conventional_mine',\n",
       " 'habitation',\n",
       " 'selective_logging',\n",
       " 'primary']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_to_label(ix_to_word, output2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sequence prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.240506Z",
     "start_time": "2017-05-08T09:40:36.238349Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.246692Z",
     "start_time": "2017-05-08T09:40:36.241852Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seq_vocab = ['clear', 'cloudy', 'haze','partly_cloudy',\n",
    "    'agriculture','artisinal_mine','bare_ground','blooming',\n",
    "    'blow_down','conventional_mine','cultivation','habitation',\n",
    "    'primary','road','selective_logging','slash_burn','water'\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.256383Z",
     "start_time": "2017-05-08T09:40:36.248055Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_batch_sequences(n, seq_vocab):\n",
    "    batch = []\n",
    "    for _ in range(n):\n",
    "        seq = ['<BEGIN>']\n",
    "        for _ in range(random.randint(1,16)):\n",
    "            seq.append(random.choice(seq_vocab))\n",
    "        seq.append('<STOP>')\n",
    "        seq = list(dict.fromkeys(seq)) # Remove duplicate while keeping order\n",
    "        batch.append(seq)\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.261197Z",
     "start_time": "2017-05-08T09:40:36.257715Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_seq = gen_batch_sequences(10, seq_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.267934Z",
     "start_time": "2017-05-08T09:40:36.262580Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['<BEGIN>', 'bare_ground', '<STOP>'],\n",
       " ['<BEGIN>',\n",
       "  'water',\n",
       "  'clear',\n",
       "  'primary',\n",
       "  'blooming',\n",
       "  'haze',\n",
       "  'cultivation',\n",
       "  'selective_logging',\n",
       "  'artisinal_mine',\n",
       "  'partly_cloudy',\n",
       "  'conventional_mine',\n",
       "  '<STOP>'],\n",
       " ['<BEGIN>', 'artisinal_mine', '<STOP>'],\n",
       " ['<BEGIN>',\n",
       "  'selective_logging',\n",
       "  'haze',\n",
       "  'conventional_mine',\n",
       "  'blooming',\n",
       "  'agriculture',\n",
       "  'artisinal_mine',\n",
       "  'blow_down',\n",
       "  'slash_burn',\n",
       "  'habitation',\n",
       "  '<STOP>'],\n",
       " ['<BEGIN>', 'primary', '<STOP>'],\n",
       " ['<BEGIN>', 'artisinal_mine', 'haze', '<STOP>'],\n",
       " ['<BEGIN>',\n",
       "  'habitation',\n",
       "  'selective_logging',\n",
       "  'water',\n",
       "  'agriculture',\n",
       "  'blooming',\n",
       "  'haze',\n",
       "  '<STOP>'],\n",
       " ['<BEGIN>',\n",
       "  'cultivation',\n",
       "  'haze',\n",
       "  'partly_cloudy',\n",
       "  'road',\n",
       "  'conventional_mine',\n",
       "  'artisinal_mine',\n",
       "  'clear',\n",
       "  '<STOP>'],\n",
       " ['<BEGIN>', 'habitation', '<STOP>'],\n",
       " ['<BEGIN>', 'habitation', '<STOP>']]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.272867Z",
     "start_time": "2017-05-08T09:40:36.269273Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seqlabels_to_target(word_to_ix, list_seq):\n",
    "    return list(map(lambda s: torch.LongTensor(\n",
    "                list(map(lambda label: word_to_ix[label], s))\n",
    "            ), list_seq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.277745Z",
     "start_time": "2017-05-08T09:40:36.274208Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeds = nn.Embedding(len(vocab), 5, padding_idx=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.282546Z",
     "start_time": "2017-05-08T09:40:36.279151Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seq_tensors = seqlabels_to_target(word_to_ix, batch_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.288284Z",
     "start_time": "2017-05-08T09:40:36.283915Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "  0\n",
       " 18\n",
       "  2\n",
       " 14\n",
       "  9\n",
       "  4\n",
       " 12\n",
       " 16\n",
       "  7\n",
       "  5\n",
       " 11\n",
       "  1\n",
       "[torch.LongTensor of size 12]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Variable(seq_tensors[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.293579Z",
     "start_time": "2017-05-08T09:40:36.289659Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.0000  0.0000  0.0000  0.0000  0.0000\n",
       " 0.0506 -1.3719 -0.2585 -0.1785  0.6526\n",
       "-1.0919  0.1947 -0.1726  1.1912  1.6100\n",
       " 1.3168 -0.0635 -1.2732  0.4671 -0.8394\n",
       " 0.3536  1.1308  1.1159  1.6579  2.1703\n",
       "-1.5680 -0.1476 -0.8719 -0.3190  0.7808\n",
       " 0.7385 -0.7012 -0.6762 -1.0369  0.2994\n",
       "-1.0733 -1.7238 -1.3880 -0.3421  1.5200\n",
       " 0.0005  0.5236  1.6549 -0.5829  0.3660\n",
       " 0.2306  1.4325  0.7247 -0.7757 -2.6264\n",
       "-0.0849 -2.8016  1.3666  1.0155 -0.1480\n",
       " 1.2792 -0.4146 -0.3290 -0.3934 -0.2931\n",
       "[torch.FloatTensor of size 12x5]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeds(Variable(seq_tensors[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.303664Z",
     "start_time": "2017-05-08T09:40:36.294606Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "  0\n",
       " 16\n",
       "  4\n",
       " 11\n",
       "  9\n",
       "  6\n",
       "  7\n",
       " 10\n",
       " 17\n",
       " 13\n",
       "  1\n",
       "[torch.LongTensor of size 11]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Variable(seq_tensors[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.309078Z",
     "start_time": "2017-05-08T09:40:36.305024Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.0000  0.0000  0.0000  0.0000  0.0000\n",
       "-1.0733 -1.7238 -1.3880 -0.3421  1.5200\n",
       "-1.5680 -0.1476 -0.8719 -0.3190  0.7808\n",
       "-0.0849 -2.8016  1.3666  1.0155 -0.1480\n",
       " 0.3536  1.1308  1.1159  1.6579  2.1703\n",
       "-0.2308 -0.8565 -1.3905  0.0961  0.1172\n",
       " 0.0005  0.5236  1.6549 -0.5829  0.3660\n",
       " 0.8778 -1.2870 -0.0061 -1.4612  2.4435\n",
       "-0.1125  1.6002  0.9264  0.1641  2.1513\n",
       "-0.6048 -0.6636 -0.3441 -1.0603  2.2499\n",
       " 1.2792 -0.4146 -0.3290 -0.3934 -0.2931\n",
       "[torch.FloatTensor of size 11x5]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeds(Variable(seq_tensors[3]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Seems like it will be a pain to work with variable size input. It would need my custom data loader. Hence I would create a custom one directly for Amazon dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.472776Z",
     "start_time": "2017-05-08T09:40:36.310273Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.utils.data.dataset import Dataset\n",
    "from torchvision import transforms\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from torch import np, from_numpy # Numpy like wrapper\n",
    "\n",
    "class TagsDataset(Dataset):\n",
    "    \"\"\"Dataset wrapping target labels for Kaggle - Planet Amazon from Space competition.\n",
    "\n",
    "    Arguments:\n",
    "        A CSV file path\n",
    "        Path to image folder\n",
    "        Extension of images\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, csv_path, vocab_mapping):\n",
    "    \n",
    "        self.df = pd.read_csv(csv_path)\n",
    "        self.vocab_mapping = vocab_mapping\n",
    "\n",
    "        self.tags = self.df['tags'].str.split()\n",
    "\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        vocab = self.vocab_mapping\n",
    "        tags = []\n",
    "        tags.append(vocab['<BEGIN>'])\n",
    "        tags.extend([vocab[tag] for tag in self.tags[index]])\n",
    "        tags.append(vocab['<STOP>'])\n",
    "        \n",
    "        tags = torch.Tensor(tags)\n",
    "        \n",
    "        return tags, tags\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df.index)\n",
    "\n",
    "\n",
    "    def collate_fn(self, data):\n",
    "        \"\"\"Creates mini-batch tensors for tags with variable size\n",
    "\n",
    "        Args:\n",
    "            data: list of tuple (input, target). \n",
    "                - input: torch tensor of shape (?); variable length.\n",
    "                - target: torch tensor of same shape (?); variable length.\n",
    "        Returns:\n",
    "            inputs: torch tensor of shape (batch_size, padded_length).\n",
    "            targets: torch tensor of shape (batch_size, padded_length).\n",
    "            lengths: list; valid length for each padded tags.\n",
    "        \"\"\"\n",
    "        # Sort a data list by target length (descending order).\n",
    "        data.sort(key=lambda x: len(x[1]), reverse=True)\n",
    "        _, tags = zip(*data)\n",
    "\n",
    "        # Merge tags (from tuple of 1D tensor to 2D tensor).\n",
    "        lengths = [len(tag) for tag in tags]\n",
    "        targets = torch.zeros(len(tags), max(lengths)).long()\n",
    "        for i, tag in enumerate(tags):\n",
    "            end = lengths[i]\n",
    "            targets[i, :end] = tag[:end]        \n",
    "        return targets, targets, lengths\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.534874Z",
     "start_time": "2017-05-08T09:40:36.474378Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = TagsDataset('./data/train.csv',word_to_ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:36.538629Z",
     "start_time": "2017-05-08T09:40:36.536247Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=X_train, \n",
    "                                              batch_size=100,\n",
    "                                              shuffle=True,\n",
    "                                              num_workers=1,\n",
    "                                              collate_fn=X_train.collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:59:58.585328Z",
     "start_time": "2017-05-08T09:59:58.568624Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SeqPred(nn.Module):\n",
    "    \"\"\" Testing weight sharing + Variable Length sequence\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, vocab_size, embed_dim, num_rnn_layers):\n",
    "        super(SeqPred, self).__init__()\n",
    " \n",
    "        self.vocab_size = vocab_size\n",
    "        self.embeds = nn.Embedding(vocab_size, embed_dim) # , padding_idx=0 Ignore the <start> (0 in vocab) for gradient\n",
    "        self.rnn = nn.LSTM(embed_dim, embed_dim, num_rnn_layers, batch_first = True)\n",
    "        self.fc = nn.Linear(embed_dim, vocab_size)\n",
    "        self.n_layers = num_rnn_layers\n",
    "        \n",
    "        # link embedding and decoding weight\n",
    "        self.fc.weight = self.embeds.weight\n",
    "    \n",
    "    \n",
    "    def forward(self, tags, lengths, hidden=None):\n",
    "        embed = self.embeds(tags)\n",
    "        print(embed.size())\n",
    "        print(len(lengths))\n",
    "        packed = pack_padded_sequence(embed, lengths, batch_first=True)\n",
    "        out, hidden = self.rnn(packed, hidden)\n",
    "        out = self.fc(out.data) #Unpack PackedSeq and feed to FC\n",
    "        return out, hidden\n",
    "\n",
    "    def genTags(self, inputs, states):\n",
    "        tag_ids = []\n",
    "        inputs = self.embeds(inputs)\n",
    "        for i in range(self.vocab_size):                    # maximum sampling length\n",
    "            hiddens, states = self.rnn(inputs, states)      # (batch_size, 1, hidden_size)\n",
    "            outputs = self.fc(hiddens.squeeze(1))           # (batch_size, vocab_size)\n",
    "            # outputs = F.softmax(outputs)\n",
    "            predicted = outputs.max(1)[1]\n",
    "            tag_ids.append(predicted)\n",
    "            inputs = self.embeds(predicted)\n",
    "        tag_ids = torch.cat(tag_ids, 1)                     # (batch_size, 19)\n",
    "        return tag_ids.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:59:59.348946Z",
     "start_time": "2017-05-08T09:59:59.345300Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = SeqPred(19, 5, 2)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:00:06.408808Z",
     "start_time": "2017-05-08T10:00:00.110773Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "Train Epoch: 000 [00000/40500 (0%)]\tLoss: 3.060821\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 11, 5])\n",
      "100\n",
      "Train Epoch: 000 [10000/40500 (25%)]\tLoss: 0.821959\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "Train Epoch: 000 [20000/40500 (49%)]\tLoss: 0.356055\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 11, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "Train Epoch: 000 [30000/40500 (74%)]\tLoss: 0.155926\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "torch.Size([100, 7, 5])\n",
      "100\n",
      "torch.Size([100, 8, 5])\n",
      "100\n",
      "Train Epoch: 000 [40000/40500 (99%)]\tLoss: 0.092131\n",
      "torch.Size([100, 10, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([100, 9, 5])\n",
      "100\n",
      "torch.Size([79, 8, 5])\n",
      "79\n"
     ]
    }
   ],
   "source": [
    "epoch =0\n",
    "for batch_idx, (data, target, lengths) in enumerate(train_loader):\n",
    "    data = Variable(data)\n",
    "    target = Variable(target)\n",
    "    targets = pack_padded_sequence(target, lengths, batch_first=True)[0]\n",
    "    \n",
    "    model.zero_grad()\n",
    "    \n",
    "    outputs, _ = model(data,lengths)\n",
    "    loss = criterion(outputs, targets)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if batch_idx % 100 == 0:\n",
    "        print('Train Epoch: {:03d} [{:05d}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "            epoch, batch_idx * len(data), len(train_loader) * len(data),\n",
    "            100. * batch_idx / len(train_loader), loss.data[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:43.038462Z",
     "start_time": "2017-05-08T09:40:43.033563Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set initial states\n",
    "state = (Variable(torch.zeros(2, 1, 5)),\n",
    "             Variable(torch.zeros(2, 1, 5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:43.043559Z",
     "start_time": "2017-05-08T09:40:43.040003Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 14\n",
       "[torch.LongTensor of size 1x1]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = Variable(torch.rand(1, 1).mul(19).long(), volatile=True)\n",
    "start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:43.048510Z",
     "start_time": "2017-05-08T09:40:43.045128Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0\n",
       "[torch.LongTensor of size 1x1]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start0 = Variable(torch.zeros(1, 1).long(), volatile=True)\n",
    "start0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T09:40:43.064593Z",
     "start_time": "2017-05-08T09:40:43.050001Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       " 14\n",
       "[torch.LongTensor of size 19]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.genTags(start,state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}