ethnicolr/models/ethnicolr_keras_lstm_fl_voter_ln_five_cat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 08:19:30.746483: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2021-12-21 08:19:30.746531: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n",
      "2.5.2\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import tensorflow as tf\n",
    "print(keras.__version__)\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "      <th>name_first</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Walker</td>\n",
       "      <td>Elizabeth</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Palmer</td>\n",
       "      <td>Alton</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mc Cleod</td>\n",
       "      <td>Alicia</td>\n",
       "      <td>nh_black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Dale</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Walker</td>\n",
       "      <td>Daniel</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710353</th>\n",
       "      <td>Walters</td>\n",
       "      <td>William</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710354</th>\n",
       "      <td>Sawyer</td>\n",
       "      <td>Matthew</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710355</th>\n",
       "      <td>Thomas</td>\n",
       "      <td>Janine</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710356</th>\n",
       "      <td>Campbell</td>\n",
       "      <td>Angel</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710357</th>\n",
       "      <td>Bruner</td>\n",
       "      <td>Jeb</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13710134 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            name_last name_first      race\n",
       "0              Walker  Elizabeth  nh_white\n",
       "1              Palmer      Alton  nh_white\n",
       "2            Mc Cleod     Alicia  nh_black\n",
       "3         Scarborough       Dale  nh_white\n",
       "4              Walker     Daniel  nh_white\n",
       "...               ...        ...       ...\n",
       "13710353      Walters    William  nh_white\n",
       "13710354       Sawyer    Matthew  nh_white\n",
       "13710355       Thomas     Janine  nh_white\n",
       "13710356     Campbell      Angel     other\n",
       "13710357       Bruner        Jeb  nh_white\n",
       "\n",
       "[13710134 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "\n",
    "NGRAMS = 2\n",
    "SAMPLE = 1000000\n",
    "EPOCHS = 20\n",
    "\n",
    "# Florida voter\n",
    "df = pd.read_csv('/opt/data/fl_voterreg/fl_reg_name_race.csv.gz')\n",
    "df.dropna(subset=['name_first', 'name_last'], inplace=True)\n",
    "df['race'] = df.race.map({'native_indian': 'other', 'asian': 'asian', 'nh_black': 'nh_black', 'hispanic': 'hispanic', 'nh_white': 'nh_white', 'other': 'other', 'multi_racial': 'other', 'unknown': 'unknown'})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "      <th>name_first</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5731290</th>\n",
       "      <td>Rao</td>\n",
       "      <td>Kranthi</td>\n",
       "      <td>asian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3915605</th>\n",
       "      <td>Hossain</td>\n",
       "      <td>Tanjim</td>\n",
       "      <td>asian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>689897</th>\n",
       "      <td>Nguyen</td>\n",
       "      <td>Bao</td>\n",
       "      <td>asian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8981436</th>\n",
       "      <td>Samaroo</td>\n",
       "      <td>RYAN</td>\n",
       "      <td>asian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11674974</th>\n",
       "      <td>Ramnauth</td>\n",
       "      <td>RYAN</td>\n",
       "      <td>asian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11684362</th>\n",
       "      <td>Lewis</td>\n",
       "      <td>Brianna</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5329148</th>\n",
       "      <td>Hughs</td>\n",
       "      <td>JAMES</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7712522</th>\n",
       "      <td>Times</td>\n",
       "      <td>Douglas</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13637468</th>\n",
       "      <td>Knutson</td>\n",
       "      <td>Elizabeth</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6276957</th>\n",
       "      <td>Vigil</td>\n",
       "      <td>Judy</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         name_last name_first   race\n",
       "5731290        Rao    Kranthi  asian\n",
       "3915605    Hossain     Tanjim  asian\n",
       "689897      Nguyen        Bao  asian\n",
       "8981436    Samaroo       RYAN  asian\n",
       "11674974  Ramnauth       RYAN  asian\n",
       "...            ...        ...    ...\n",
       "11684362     Lewis    Brianna  other\n",
       "5329148      Hughs      JAMES  other\n",
       "7712522      Times    Douglas  other\n",
       "13637468   Knutson  Elizabeth  other\n",
       "6276957      Vigil       Judy  other\n",
       "\n",
       "[1000000 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdf = df[df.race.isin(['unknown']) == False].groupby(['race']).sample(int(SAMPLE/5), random_state=21)\n",
    "del df\n",
    "\n",
    "# Additional features\n",
    "sdf['name_last'] = sdf.name_last.str.title()\n",
    "sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>200000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name_last\n",
       "race               \n",
       "asian        200000\n",
       "hispanic     200000\n",
       "nh_black     200000\n",
       "nh_white     200000\n",
       "other        200000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdf = sdf.groupby('race').agg({'name_last': 'count'})\n",
    "rdf.to_csv('./fl_voter_reg/lstm/fl_ln_five_cat_race.csv', columns=[])\n",
    "rdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>49168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>47270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>28558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>64985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>65497</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name_last\n",
       "race               \n",
       "asian         49168\n",
       "hispanic      47270\n",
       "nh_black      28558\n",
       "nh_white      64985\n",
       "other         65497"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdf.groupby('race').agg({'name_last': 'nunique'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_words = 1196\n",
      "Max feature len = 27, Avg. feature len = 5\n"
     ]
    }
   ],
   "source": [
    "# last name only\n",
    "sdf['name_last_name_first'] = sdf['name_last']\n",
    "\n",
    "# build n-gram list\n",
    "vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=3, ngram_range=(NGRAMS, NGRAMS), lowercase=False) \n",
    "a = vect.fit_transform(sdf.name_last_name_first)\n",
    "vocab = vect.vocabulary_\n",
    "\n",
    "# sort n-gram by freq (highest -> lowest)\n",
    "words = []\n",
    "for b in vocab:\n",
    "    c = vocab[b]\n",
    "    #print(b, c, a[:, c].sum())\n",
    "    words.append((a[:, c].sum(), b))\n",
    "    #break\n",
    "words = sorted(words, reverse=True)\n",
    "words_list = ['UNK']\n",
    "words_list.extend([w[1] for w in words])\n",
    "num_words = len(words_list)\n",
    "print(\"num_words = %d\" % num_words)\n",
    "\n",
    "\n",
    "def find_ngrams(text, n):\n",
    "    a = zip(*[text[i:] for i in range(n)])\n",
    "    wi = []\n",
    "    for i in a:\n",
    "        w = ''.join(i)\n",
    "        try:\n",
    "            idx = words_list.index(w)\n",
    "        except:\n",
    "            idx = 0\n",
    "        wi.append(idx)\n",
    "    return wi\n",
    "\n",
    "# build X from index of n-gram sequence\n",
    "X = np.array(sdf.name_last_name_first.apply(lambda c: find_ngrams(c, NGRAMS)))\n",
    "\n",
    "# check max/avg feature\n",
    "X_len = []\n",
    "for x in X:\n",
    "    X_len.append(len(x))\n",
    "\n",
    "max_feature_len = max(X_len)\n",
    "avg_feature_len = int(np.mean(X_len))\n",
    "\n",
    "print(\"Max feature len = %d, Avg. feature len = %d\" % (max_feature_len, avg_feature_len))\n",
    "y = np.array(sdf.race.astype('category').cat.codes)\n",
    "\n",
    "# Split train and test dataset\n",
    "X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a LSTM model\n",
    "\n",
    "ref: http://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800000 train sequences\n",
      "200000 test sequences\n",
      "Pad sequences (samples x time)\n",
      "X_train shape: (800000, 20)\n",
      "X_test shape: (200000, 20)\n",
      "5 classes\n",
      "Convert class vector to binary class matrix (for use with categorical_crossentropy)\n",
      "y_train shape: (800000, 5)\n",
      "y_test shape: (200000, 5)\n"
     ]
    }
   ],
   "source": [
    "'''The dataset is actually too small for LSTM to be of any advantage\n",
    "compared to simpler, much faster methods such as TF-IDF + LogReg.\n",
    "Notes:\n",
    "\n",
    "- RNNs are tricky. Choice of batch size is important,\n",
    "choice of loss and optimizer is critical, etc.\n",
    "Some configurations won't converge.\n",
    "\n",
    "- LSTM loss decrease patterns during training can be quite different\n",
    "from what you see with CNNs/MLPs/etc.\n",
    "'''\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, Dropout, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "from keras.models import load_model\n",
    "\n",
    "max_features = num_words # 20000\n",
    "feature_len = 20 # avg_feature_len # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 32\n",
    "\n",
    "print(len(X_train), 'train sequences')\n",
    "print(len(X_test), 'test sequences')\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=feature_len)\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=feature_len)\n",
    "print('X_train shape:', X_train.shape)\n",
    "print('X_test shape:', X_test.shape)\n",
    "\n",
    "num_classes = np.max(y_train) + 1\n",
    "print(num_classes, 'classes')\n",
    "\n",
    "print('Convert class vector to binary class matrix '\n",
    "      '(for use with categorical_crossentropy)')\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 20, 32)            38272     \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 128)               82432     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 5)                 645       \n",
      "=================================================================\n",
      "Total params: 121,349\n",
      "Trainable params: 121,349\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 08:21:05.800031: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2021-12-21 08:21:05.800072: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-12-21 08:21:05.800090: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (jupyter-server-2): /proc/driver/nvidia/version does not exist\n",
      "2021-12-21 08:21:05.800295: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(num_words, 32, input_length=feature_len))\n",
    "model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 08:21:06.163286: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:176] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2021-12-21 08:21:06.163898: I tensorflow/core/platform/profile_utils/cpu_utils.cc:114] CPU Frequency: 2250000000 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "22500/22500 [==============================] - 458s 19ms/step - loss: 1.2485 - accuracy: 0.4808 - val_loss: 1.1686 - val_accuracy: 0.5246\n",
      "Epoch 2/20\n",
      "22500/22500 [==============================] - 428s 19ms/step - loss: 1.1646 - accuracy: 0.5261 - val_loss: 1.1404 - val_accuracy: 0.5410\n",
      "Epoch 3/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.1400 - accuracy: 0.5406 - val_loss: 1.1292 - val_accuracy: 0.5466\n",
      "Epoch 4/20\n",
      "22500/22500 [==============================] - 417s 19ms/step - loss: 1.1266 - accuracy: 0.5486 - val_loss: 1.1216 - val_accuracy: 0.5513\n",
      "Epoch 5/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.1203 - accuracy: 0.5514 - val_loss: 1.1146 - val_accuracy: 0.5570\n",
      "Epoch 6/20\n",
      "22500/22500 [==============================] - 417s 19ms/step - loss: 1.1138 - accuracy: 0.5561 - val_loss: 1.1129 - val_accuracy: 0.5576\n",
      "Epoch 7/20\n",
      "22500/22500 [==============================] - 416s 18ms/step - loss: 1.1107 - accuracy: 0.5574 - val_loss: 1.1101 - val_accuracy: 0.5589\n",
      "Epoch 8/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.1075 - accuracy: 0.5588 - val_loss: 1.1094 - val_accuracy: 0.5606\n",
      "Epoch 9/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.1039 - accuracy: 0.5605 - val_loss: 1.1054 - val_accuracy: 0.5612\n",
      "Epoch 10/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.1041 - accuracy: 0.5596 - val_loss: 1.1048 - val_accuracy: 0.5618\n",
      "Epoch 11/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.1013 - accuracy: 0.5627 - val_loss: 1.1032 - val_accuracy: 0.5651\n",
      "Epoch 12/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.1012 - accuracy: 0.5628 - val_loss: 1.1042 - val_accuracy: 0.5642\n",
      "Epoch 13/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.0998 - accuracy: 0.5631 - val_loss: 1.1017 - val_accuracy: 0.5636\n",
      "Epoch 14/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.0996 - accuracy: 0.5634 - val_loss: 1.1013 - val_accuracy: 0.5639\n",
      "Epoch 15/20\n",
      "22500/22500 [==============================] - 420s 19ms/step - loss: 1.0995 - accuracy: 0.5630 - val_loss: 1.1028 - val_accuracy: 0.5648\n",
      "Epoch 16/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.0966 - accuracy: 0.5646 - val_loss: 1.1006 - val_accuracy: 0.5648\n",
      "Epoch 17/20\n",
      "22500/22500 [==============================] - 418s 19ms/step - loss: 1.0991 - accuracy: 0.5626 - val_loss: 1.0990 - val_accuracy: 0.5654\n",
      "Epoch 18/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.0963 - accuracy: 0.5650 - val_loss: 1.1002 - val_accuracy: 0.5648\n",
      "Epoch 19/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.0981 - accuracy: 0.5645 - val_loss: 1.0996 - val_accuracy: 0.5657\n",
      "Epoch 20/20\n",
      "22500/22500 [==============================] - 419s 19ms/step - loss: 1.0944 - accuracy: 0.5652 - val_loss: 1.0999 - val_accuracy: 0.5664\n",
      "6250/6250 [==============================] - 28s 5ms/step - loss: 1.0941 - accuracy: 0.5679\n",
      "Test score: 1.094052791595459\n",
      "Test accuracy: 0.5678849816322327\n"
     ]
    }
   ],
   "source": [
    "print('Train...')\n",
    "model.fit(X_train, y_train, batch_size=batch_size, epochs=EPOCHS,\n",
    "          validation_split=0.1, verbose=1)\n",
    "score, acc = model.evaluate(X_test, y_test,\n",
    "                            batch_size=batch_size, verbose=1)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 1.094052791595459\n",
      "Test accuracy: 0.5678849816322327\n"
     ]
    }
   ],
   "source": [
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6250/6250 - 27s\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       asian       0.71      0.63      0.67     40000\n",
      "    hispanic       0.72      0.83      0.77     40000\n",
      "    nh_black       0.49      0.78      0.60     40000\n",
      "    nh_white       0.44      0.53      0.48     40000\n",
      "       other       0.38      0.06      0.11     40000\n",
      "\n",
      "    accuracy                           0.57    200000\n",
      "   macro avg       0.55      0.57      0.53    200000\n",
      "weighted avg       0.55      0.57      0.53    200000\n",
      "\n",
      "[[25339  3753  3974  5391  1543]\n",
      " [ 1313 33307  1754  2983   643]\n",
      " [  945   834 31071  6463   687]\n",
      " [ 1698  2414 13220 21288  1380]\n",
      " [ 6515  5687 13248 11978  2572]]\n"
     ]
    }
   ],
   "source": [
    "p = model.predict(X_test, verbose=2) # to predict probability\n",
    "y_pred = np.argmax(p, axis=-1)\n",
    "target_names = list(sdf.race.astype('category').cat.categories)\n",
    "print(classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names))\n",
    "print(confusion_matrix(np.argmax(y_test, axis=1), y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./fl_voter_reg/lstm/fl_all_ln_lstm_5_cat.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_df = pd.DataFrame(words_list, columns=['vocab'])\n",
    "words_df.to_csv('./fl_voter_reg/lstm/fl_all_ln_vocab_5_cat.csv', index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}