Dual_Feed_Image_Label.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.499144Z",
     "start_time": "2017-05-08T10:52:32.352476Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch import nn, ones\n",
    "from torch.autograd import Variable\n",
    "from torchvision import models\n",
    "from torch.nn.init import kaiming_normal\n",
    "from torch import np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import random\n",
    "import numpy as np\n",
    "from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence\n",
    "from src.p_data_augmentation import PowerPIL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.680830Z",
     "start_time": "2017-05-08T10:52:32.500741Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.utils.data.dataset import Dataset\n",
    "from torchvision import transforms\n",
    "import pandas as pd\n",
    "import os\n",
    "from PIL import Image\n",
    "\n",
    "from torch import np, from_numpy # Numpy like wrapper\n",
    "\n",
    "class ImgTagsDualFeedDataset(Dataset):\n",
    "    \"\"\"Dataset wrapping images, labels and target labels for Kaggle - Planet Amazon from Space competition.\n",
    "\n",
    "    Arguments:\n",
    "        A CSV file path\n",
    "        Path to image folder\n",
    "        Extension of images\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, csv_path, img_path, img_ext, vocab_mapping, transform=None):\n",
    "    \n",
    "        self.df = pd.read_csv(csv_path)\n",
    "        assert self.df['image_name'].apply(lambda x: os.path.isfile(img_path + x + img_ext)).all(), \\\n",
    "\"Some images referenced in the CSV file were not found\"\n",
    "\n",
    "        self.img_path = img_path\n",
    "        self.img_ext = img_ext\n",
    "        self.transform = transform\n",
    "        \n",
    "        self.X = self.df['image_name']\n",
    "        \n",
    "        self.vocab_mapping = vocab_mapping\n",
    "\n",
    "        self.tags = self.df['tags'].str.split()\n",
    "        \n",
    "        \n",
    "    def X(self):\n",
    "        return self.X\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        \n",
    "        img = Image.open(self.img_path + self.X[index] + self.img_ext)\n",
    "        img = img.convert('RGB')\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "            \n",
    "        vocab = self.vocab_mapping\n",
    "        tags = []\n",
    "        tags.append(vocab['<BEGIN>'])\n",
    "        tags.extend([vocab[tag] for tag in self.tags[index]])\n",
    "        tags.append(vocab['<STOP>'])\n",
    "        \n",
    "        tags = torch.Tensor(tags)\n",
    "        return img, tags\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df.index)\n",
    "\n",
    "\n",
    "    def collate_fn(self, data):\n",
    "        \"\"\"Creates mini-batch tensors for tags with variable size\n",
    "\n",
    "        Args:\n",
    "            data: list of tuple (input, target). \n",
    "                - image: torch tensor of shape (3, ?, ?).\n",
    "                - target: torch tensor of same shape (?); variable length.\n",
    "        Returns:\n",
    "            images: torch tensor of shape (batch_size, 3, ?, ?).\n",
    "            targets: torch tensor of shape (batch_size, padded_length).\n",
    "            lengths: list; valid length for each padded tags.\n",
    "        \"\"\"\n",
    "        # Sort a data list by target length (descending order).\n",
    "        data.sort(key=lambda x: len(x[1]), reverse=True)\n",
    "        imgs, tags = zip(*data)\n",
    "        \n",
    "        # Merge images (from tuple of 3D tensor to 4D tensor).\n",
    "        imgs = torch.stack(imgs, 0)\n",
    "\n",
    "        # Merge tags (from tuple of 1D tensor to 2D tensor).\n",
    "        lengths = [len(tag) for tag in tags]\n",
    "        targets = torch.zeros(len(tags), max(lengths)).long()\n",
    "        for i, tag in enumerate(tags):\n",
    "            end = lengths[i]\n",
    "            targets[i, :end] = tag[:end]        \n",
    "        return imgs, targets, lengths\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.685184Z",
     "start_time": "2017-05-08T10:52:32.682334Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                     std=[0.229, 0.224, 0.225])\n",
    "ds_transform_augmented = transforms.Compose([\n",
    "                     transforms.RandomSizedCrop(224),\n",
    "                     PowerPIL(),\n",
    "                     transforms.ToTensor(),\n",
    "                     normalize,\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.692774Z",
     "start_time": "2017-05-08T10:52:32.686438Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<BEGIN>': 0, '<STOP>': 1, 'clear': 2, 'cloudy': 3, 'haze': 4, 'partly_cloudy': 5, 'agriculture': 6, 'artisinal_mine': 7, 'bare_ground': 8, 'blooming': 9, 'blow_down': 10, 'conventional_mine': 11, 'cultivation': 12, 'habitation': 13, 'primary': 14, 'road': 15, 'selective_logging': 16, 'slash_burn': 17, 'water': 18}\n"
     ]
    }
   ],
   "source": [
    "vocab = ['<BEGIN>','<STOP>','clear', 'cloudy', 'haze','partly_cloudy',\n",
    "    'agriculture','artisinal_mine','bare_ground','blooming',\n",
    "    'blow_down','conventional_mine','cultivation','habitation',\n",
    "    'primary','road','selective_logging','slash_burn','water'\n",
    "    ]\n",
    "\n",
    "word_to_ix = { word: i for i, word in enumerate(vocab) }\n",
    "print(word_to_ix)\n",
    "one_hot_mapping = {k:np.eye(19)[v] for k,v in word_to_ix.items()}\n",
    "# print(one_hot_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.844270Z",
     "start_time": "2017-05-08T10:52:32.693843Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = ImgTagsDualFeedDataset('./data/train.csv','./data/train-jpg/','.jpg',\n",
    "                                 word_to_ix,\n",
    "                                 ds_transform_augmented\n",
    "                                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.848407Z",
     "start_time": "2017-05-08T10:52:32.845707Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=X_train, \n",
    "                                              batch_size=64,\n",
    "                                              shuffle=True,\n",
    "                                              num_workers=4,\n",
    "                                              pin_memory = True,\n",
    "                                              collate_fn=X_train.collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.919202Z",
     "start_time": "2017-05-08T10:52:32.849855Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       " ( 0 ,.,.) = \n",
       "  -0.8849 -0.8678 -0.7822  ...  -1.5699 -1.5699 -1.5528\n",
       "  -0.8507 -0.8335 -0.7479  ...  -1.5699 -1.5699 -1.5699\n",
       "  -0.7650 -0.7650 -0.6794  ...  -1.5699 -1.5870 -1.5870\n",
       "            ...             ⋱             ...          \n",
       "  -1.4500 -1.4500 -1.4329  ...  -1.5014 -1.5357 -1.5528\n",
       "  -1.4500 -1.4500 -1.4329  ...  -1.5185 -1.5357 -1.5528\n",
       "  -1.4500 -1.4500 -1.4329  ...  -1.5185 -1.5528 -1.5528\n",
       " \n",
       " ( 1 ,.,.) = \n",
       "  -0.7577 -0.7577 -0.7052  ...  -1.3704 -1.3529 -1.3529\n",
       "  -0.7227 -0.7227 -0.6527  ...  -1.3704 -1.3529 -1.3529\n",
       "  -0.6527 -0.6352 -0.5826  ...  -1.3704 -1.3529 -1.3529\n",
       "            ...             ⋱             ...          \n",
       "  -1.2304 -1.2304 -1.2129  ...  -1.2654 -1.3004 -1.3004\n",
       "  -1.2654 -1.2654 -1.2479  ...  -1.2654 -1.2829 -1.3004\n",
       "  -1.3004 -1.2829 -1.2654  ...  -1.2654 -1.2829 -1.3004\n",
       " \n",
       " ( 2 ,.,.) = \n",
       "  -0.7413 -0.7413 -0.7064  ...  -1.1944 -1.1770 -1.1596\n",
       "  -0.7413 -0.7413 -0.6715  ...  -1.1944 -1.1770 -1.1770\n",
       "  -0.7064 -0.6890 -0.6018  ...  -1.1596 -1.1770 -1.1770\n",
       "            ...             ⋱             ...          \n",
       "  -1.1073 -1.1073 -1.0724  ...  -1.1770 -1.1944 -1.1944\n",
       "  -1.1247 -1.1247 -1.1073  ...  -1.1596 -1.1770 -1.1770\n",
       "  -1.1421 -1.1247 -1.1247  ...  -1.1596 -1.1421 -1.1596\n",
       " [torch.FloatTensor of size 3x224x224], \n",
       "   0\n",
       "   6\n",
       "   2\n",
       "  14\n",
       "  18\n",
       "   1\n",
       " [torch.FloatTensor of size 6])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:32.960983Z",
     "start_time": "2017-05-08T10:52:32.920544Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CNN_RNN_Fused(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, num_rnn_layers):\n",
    "        super(CNN_RNN_Fused, self).__init__()\n",
    "        \n",
    "        ## CNN part\n",
    "        # Loading ResNet arch from PyTorch and weights from Pycaffe\n",
    "        original_model = models.resnet50(pretrained=False)\n",
    "        original_model.load_state_dict(torch.load('./zoo/resnet50.pth'))\n",
    "        \n",
    "        # Everything except the last linear layer\n",
    "        self.convnet = nn.Sequential(*list(original_model.children())[:-1])\n",
    "        \n",
    "        # Get number of features of last layer\n",
    "        num_feats_cnn = original_model.fc.in_features\n",
    "        \n",
    "        ## RNN part\n",
    "        hidden_size = embed_dim # for simplification\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embeds = nn.Embedding(vocab_size, embed_dim) # , padding_idx=0 Ignore the <start> (0 in vocab) for gradient\n",
    "        self.rnn = nn.LSTM(embed_dim, hidden_size, num_rnn_layers, batch_first = True)\n",
    "        self.num_rnn_layers = num_rnn_layers\n",
    "        \n",
    "        ## Projection\n",
    "        self.prj_cnn = nn.Linear(num_feats_cnn, embed_dim)\n",
    "        self.prj_rnn = nn.Linear(hidden_size, embed_dim)\n",
    "        \n",
    "\n",
    "        ## Prediction\n",
    "        # link embedding and decoding weight\n",
    "        self.fc = nn.Linear(embed_dim, vocab_size)\n",
    "        self.fc.weight = self.embeds.weight\n",
    "    \n",
    "    \n",
    "    def forward(self, img, tags, lengths, hidden=None):\n",
    "        ## CNN\n",
    "        cnn_feats = self.convnet(img)\n",
    "        cnn_feats = cnn_feats.view(cnn_feats.size(0), -1)\n",
    "        cnn_feats = self.prj_cnn(cnn_feats)\n",
    "        \n",
    "        tag_ids = []\n",
    "        embed = self.embeds(tags)\n",
    "        for _ in tags:\n",
    "            ## RNN\n",
    "            rnn_out, hidden = self.rnn(embed, hidden)\n",
    "\n",
    "            ## Projection\n",
    "            rnn_out = self.prj_rnn(rnn_out[:,0,:]) # Extract the first prediction from sequence\n",
    "            fuse = cnn_feats + rnn_out\n",
    "            fuse = self.fc(fuse)\n",
    "            predicted = fuse.max(1)[1]\n",
    "            tag_ids.append(predicted)\n",
    "            packed = self.embeds(predicted)\n",
    "        tag_ids = torch.cat(tag_ids, 1)\n",
    "        print(tag_ids)\n",
    "        return tag_ids.squeeze()\n",
    "\n",
    "    def genTags(self, inputs, states=None):\n",
    "        tag_ids = []\n",
    "        inputs = self.embeds(inputs)\n",
    "        for i in range(self.vocab_size):                    # maximum sampling length\n",
    "            hiddens, states = self.rnn(inputs, states)      # (batch_size, 1, hidden_size)\n",
    "            outputs = self.fc(hiddens.squeeze(1))           # (batch_size, vocab_size)\n",
    "            # outputs = F.softmax(outputs)\n",
    "            predicted = outputs.max(1)[1]\n",
    "            tag_ids.append(predicted)\n",
    "            inputs = self.embeds(predicted)\n",
    "        tag_ids = torch.cat(tag_ids, 1)                     # (batch_size, 19)\n",
    "        return tag_ids.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:35.557189Z",
     "start_time": "2017-05-08T10:52:32.962267Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = CNN_RNN_Fused(19, 5, 2).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:35.561880Z",
     "start_time": "2017-05-08T10:52:35.559070Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-08T10:52:37.039622Z",
     "start_time": "2017-05-08T10:52:35.563145Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      "   10    10    10  ...     10    10    10\n",
      "   10    10    10  ...     10    10    10\n",
      "    0     0     0  ...      0     0     0\n",
      "       ...          ⋱          ...       \n",
      "    0    16    16  ...     16    16    16\n",
      "   10    10    10  ...     10    10    10\n",
      "   10    10    10  ...     10    10    10\n",
      "[torch.cuda.LongTensor of size 64x64 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  0\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  7\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  6\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  5\n",
      "  2\n",
      "  5\n",
      "  2\n",
      "  5\n",
      "  5\n",
      "  2\n",
      "  4\n",
      "  2\n",
      "  5\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  5\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  5\n",
      "  5\n",
      "  4\n",
      "  5\n",
      "  2\n",
      "  4\n",
      "  5\n",
      "  5\n",
      "  2\n",
      "  4\n",
      "  4\n",
      "  4\n",
      "  2\n",
      "  4\n",
      "  2\n",
      "  2\n",
      "  3\n",
      "  3\n",
      "  3\n",
      "  3\n",
      "  3\n",
      "  2\n",
      "  8\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  5\n",
      " 12\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  2\n",
      "  5\n",
      "  2\n",
      "  4\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 14\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      " 12\n",
      "  2\n",
      " 13\n",
      " 14\n",
      " 13\n",
      " 13\n",
      " 14\n",
      " 14\n",
      "  5\n",
      " 12\n",
      " 14\n",
      " 14\n",
      " 12\n",
      " 12\n",
      " 14\n",
      " 14\n",
      " 14\n",
      " 18\n",
      " 18\n",
      " 18\n",
      " 18\n",
      " 18\n",
      " 18\n",
      " 16\n",
      " 15\n",
      " 18\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      " 13\n",
      " 13\n",
      " 14\n",
      " 15\n",
      " 14\n",
      " 14\n",
      " 15\n",
      " 15\n",
      " 14\n",
      " 14\n",
      " 18\n",
      " 18\n",
      " 14\n",
      " 14\n",
      " 15\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      " 14\n",
      " 14\n",
      " 15\n",
      " 18\n",
      " 15\n",
      " 15\n",
      " 18\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      " 15\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      "  1\n",
      " 18\n",
      "  1\n",
      "[torch.cuda.LongTensor of size 301 (GPU 0)]\n",
      "\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "<class 'torch.cuda.LongTensor'>",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-3813a9be8c09>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;31m# check one tag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/torch/nn/modules/loss.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m    319\u001b[0m         \u001b[0m_assert_no_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m         return F.cross_entropy(input, target,\n\u001b[0;32m--> 321\u001b[0;31m                                self.weight, self.size_average)\n\u001b[0m\u001b[1;32m    322\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mcross_entropy\u001b[0;34m(input, target, weight, size_average)\u001b[0m\n\u001b[1;32m    533\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0meach\u001b[0m \u001b[0mminibatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m     \"\"\"\n\u001b[0;32m--> 535\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize_average\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    536\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlog_softmax\u001b[0;34m(input)\u001b[0m\n\u001b[1;32m    434\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    435\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 436\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLogSoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    437\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/torch/nn/_functions/thnn/auto.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, *params)\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 110\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype2backend\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mparam\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/torch/_thnn/__init__.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackends\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: <class 'torch.cuda.LongTensor'>"
     ]
    }
   ],
   "source": [
    "epoch =0\n",
    "for batch_idx, (img, tags, lengths) in enumerate(train_loader):\n",
    "    img = Variable(img).cuda()\n",
    "    tags = Variable(tags).cuda()\n",
    "    targets = pack_padded_sequence(tags, lengths, batch_first=True)[0]\n",
    "\n",
    "    \n",
    "    model.zero_grad()\n",
    "    \n",
    "    # Predict one tag at a time\n",
    "    outputs = model(img, tags, lengths)\n",
    "    \n",
    "    # check one tag\n",
    "    print(targets)\n",
    "    loss = criterion(outputs, targets)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if batch_idx % 100 == 0:\n",
    "        print('Train Epoch: {:03d} [{:05d}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "            epoch, batch_idx * len(data), len(train_loader) * len(data),\n",
    "            100. * batch_idx / len(train_loader), loss.data[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}