github-bugs-prediction-using-transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Link: https://www.kaggle.com/anmolkumar/github-bugs-prediction/download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: BeautifulSoup4 in c:\\users\\kaif\\anaconda3\\envs\\opencv_project\\lib\\site-packages (4.10.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\kaif\\anaconda3\\envs\\opencv_project\\lib\\site-packages (from BeautifulSoup4) (2.3.1)\n",
      "Requirement already satisfied: pydot in c:\\users\\kaif\\anaconda3\\envs\\opencv_project\\lib\\site-packages (1.4.2)\n",
      "Requirement already satisfied: pyparsing>=2.1.4 in c:\\users\\kaif\\anaconda3\\envs\\opencv_project\\lib\\site-packages (from pydot) (3.0.7)\n"
     ]
    }
   ],
   "source": [
    "!pip install BeautifulSoup4\n",
    "!pip install pydot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import Counter,defaultdict\n",
    "import gc\n",
    "import string\n",
    "from unidecode import unidecode\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.patches as mpatches\n",
    "import matplotlib\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.stem import WordNetLemmatizer,PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.layers import LSTM, Dense\n",
    "#from keras.optimizers import Adam\n",
    "import keras.backend as k\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.layers import Input, Embedding, Dense, Concatenate, TimeDistributed, Bidirectional,GRU\n",
    "from tensorflow.keras.models import Model,Sequential\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "import transformers\n",
    "from tqdm.notebook import tqdm\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "from transformers import AutoTokenizer, pipeline, TFDistilBertModel,TFAutoModel\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' reading files '''\n",
    "train = pd.read_json(\"D:/NLP Projects 33 - GitHub Bugs Prediction/archive/embold_train.json\").reset_index(drop=True)\n",
    "test = pd.read_json(\"D:/NLP Projects 33 - GitHub Bugs Prediction/archive/embold_test.json\").reset_index(drop=True)\n",
    "train_extra= pd.read_json(\"D:/NLP Projects 33 - GitHub Bugs Prediction/archive/embold_train_extra.json\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func(x):\n",
    "    return x['title'] + \" \" + x['body']   \n",
    "\n",
    "train['text'] = train.apply(lambda x : func(x),axis=1)\n",
    "test['text'] = test.apply(lambda x : func(x),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' creating dictionary '''\n",
    "cList = {\n",
    "        \"i'm\": \"i am\",\n",
    "        \"you're\": \"you are\",\n",
    "        \"it's\": \"it is\",\n",
    "        \"we're\": \"we are\",\n",
    "        \"we'll\": \"we will\",\n",
    "        \"That's\":\"that is\",\n",
    "        \"haven't\":\"have not\",\n",
    "        \"let's\":\"let us\",\n",
    "        \"ain't\": \"am not / are not / is not / has not / have not\",\n",
    "        \"aren't\": \"are not / am not\",\n",
    "        \"can't\": \"cannot\",\n",
    "        \"can't've\": \"cannot have\",\n",
    "        \"'cause\": \"because\",\n",
    "        \"could've\": \"could have\",\n",
    "        \"couldn't\": \"could not\",\n",
    "        \"couldn't've\": \"could not have\",\n",
    "        \"didn't\": \"did not\",\n",
    "        \"doesn't\": \"does not\",\n",
    "        \"don't\": \"do not\",\n",
    "        \"hadn't\": \"had not\",\n",
    "        \"hadn't've\": \"had not have\",\n",
    "        \"hasn't\": \"has not\",\n",
    "        \"haven't\": \"have not\",\n",
    "        \"he'd\": \"he had / he would\",\n",
    "        \"he'd've\": \"he would have\",\n",
    "        \"he'll\": \"he shall / he will\",\n",
    "        \"he'll've\": \"he shall have / he will have\",\n",
    "        \"he's\": \"he has / he is\",\n",
    "        \"how'd\": \"how did\",\n",
    "        \"how'd'y\": \"how do you\",\n",
    "        \"how'll\": \"how will\",\n",
    "        \"how's\": \"how has / how is / how does\",\n",
    "        \"I'd\": \"I had / I would\",\n",
    "        \"I'd've\": \"I would have\",\n",
    "        \"I'll\": \"I shall / I will\",\n",
    "        \"I'll've\": \"I shall have / I will have\",\n",
    "        \"I'm\": \"I am\",\n",
    "        \"I've\": \"I have\",\n",
    "        \"isn't\": \"is not\",\n",
    "        \"it'd\": \"it had / it would\",\n",
    "        \"it'd've\": \"it would have\",\n",
    "        \"it'll\": \"it shall / it will\",\n",
    "        \"it'll've\": \"it shall have / it will have\",\n",
    "        \"it's\": \"it has / it is\",\n",
    "        \"let's\": \"let us\",\n",
    "        \"ma'am\": \"madam\",\n",
    "        \"mayn't\": \"may not\",\n",
    "        \"might've\": \"might have\",\n",
    "        \"mightn't\": \"might not\",\n",
    "        \"mightn't've\": \"might not have\",\n",
    "        \"must've\": \"must have\",\n",
    "        \"mustn't\": \"must not\",\n",
    "        \"mustn't've\": \"must not have\",\n",
    "        \"needn't\": \"need not\",\n",
    "        \"needn't've\": \"need not have\",\n",
    "        \"o'clock\": \"of the clock\",\n",
    "        \"oughtn't\": \"ought not\",\n",
    "        \"oughtn't've\": \"ought not have\",\n",
    "        \"shan't\": \"shall not\",\n",
    "        \"sha'n't\": \"shall not\",\n",
    "        \"shan't've\": \"shall not have\",\n",
    "        \"she'd\": \"she had / she would\",\n",
    "        \"she'd've\": \"she would have\",\n",
    "        \"she'll\": \"she shall / she will\",\n",
    "        \"she'll've\": \"she shall have / she will have\",\n",
    "        \"she's\": \"she has / she is\",\n",
    "        \"should've\": \"should have\",\n",
    "        \"shouldn't\": \"should not\",\n",
    "        \"shouldn't've\": \"should not have\",\n",
    "        \"so've\": \"so have\",\n",
    "        \"so's\": \"so as / so is\",\n",
    "        \"that'd\": \"that would / that had\",\n",
    "        \"that'd've\": \"that would have\",\n",
    "        \"that's\": \"that has / that is\",\n",
    "        \"there'd\": \"there had / there would\",\n",
    "        \"there'd've\": \"there would have\",\n",
    "        \"there's\": \"there has / there is\",\n",
    "        \"they'd\": \"they had / they would\",\n",
    "        \"they'd've\": \"they would have\",\n",
    "        \"they'll\": \"they shall / they will\",\n",
    "        \"they'll've\": \"they shall have / they will have\",\n",
    "        \"they're\": \"they are\",\n",
    "        \"they've\": \"they have\",\n",
    "        \"to've\": \"to have\",\n",
    "        \"wasn't\": \"was not\",\n",
    "        \"we'd\": \"we had / we would\",\n",
    "        \"we'd've\": \"we would have\",\n",
    "        \"we'll\": \"we will\",\n",
    "        \"we'll've\": \"we will have\",\n",
    "        \"we're\": \"we are\",\n",
    "        \"we've\": \"we have\",\n",
    "        \"weren't\": \"were not\",\n",
    "        \"what'll\": \"what shall / what will\",\n",
    "        \"what'll've\": \"what shall have / what will have\",\n",
    "        \"what're\": \"what are\",\n",
    "        \"what's\": \"what has / what is\",\n",
    "        \"what've\": \"what have\",\n",
    "        \"when's\": \"when has / when is\",\n",
    "        \"when've\": \"when have\",\n",
    "        \"where'd\": \"where did\",\n",
    "        \"where's\": \"where has / where is\",\n",
    "        \"where've\": \"where have\",\n",
    "        \"who'll\": \"who shall / who will\",\n",
    "        \"who'll've\": \"who shall have / who will have\",\n",
    "        \"who's\": \"who has / who is\",\n",
    "        \"who've\": \"who have\",\n",
    "        \"why's\": \"why has / why is\",\n",
    "        \"why've\": \"why have\",\n",
    "        \"will've\": \"will have\",\n",
    "        \"won't\": \"will not\",\n",
    "        \"won't've\": \"will not have\",\n",
    "        \"would've\": \"would have\",\n",
    "        \"wouldn't\": \"would not\",\n",
    "        \"wouldn't've\": \"would not have\",\n",
    "        \"y'all\": \"you all\",\n",
    "        \"y'all'd\": \"you all would\",\n",
    "        \"y'all'd've\": \"you all would have\",\n",
    "        \"y'all're\": \"you all are\",\n",
    "        \"y'all've\": \"you all have\",\n",
    "        \"you'd\": \"you had / you would\",\n",
    "        \"you'd've\": \"you would have\",\n",
    "        \"you'll\": \"you shall / you will\",\n",
    "        \"you'll've\": \"you shall have / you will have\",\n",
    "        \"you're\": \"you are\",\n",
    "        \"you've\": \"you have\"\n",
    "   }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' regex '''\n",
    "c_re = re.compile('(%s)' % '|'.join(cList.keys()))\n",
    "\n",
    "def expand_contractions(text, c_re=c_re):\n",
    "    def replace(match):\n",
    "        return cList[match.group(0)]\n",
    "    return c_re.sub(replace, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeEmoji(s):\n",
    "        emoji_p = re.compile(\"[\"\n",
    "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                               u\"\\U00002702-\\U000027B0\"\n",
    "                               u\"\\U000024C2-\\U0001F251\"\n",
    "                               \"]+\", flags=re.UNICODE)\n",
    "        return emoji_p.sub(r'', s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removePunctuations(data):\n",
    "    p_tag = re.compile(r'[^\\w\\s]')\n",
    "    d = p_tag.sub(r'',data)\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_special_chars(d):\n",
    "    '''\n",
    "    Removes special characters which are specifically found in tweets.\n",
    "    '''\n",
    "    \n",
    "    ''' Creating HTML tags '''\n",
    "    beau_soup = BeautifulSoup(d, \"html.parser\")\n",
    "    d = beau_soup.get_text()\n",
    "\n",
    "    '''Convert www.* or https?://* to empty strings'''\n",
    "    d = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+))','',d)\n",
    "\n",
    "    '''Convert @username to empty strings'''\n",
    "    d = re.sub('@[^\\s]+','',d)\n",
    "    \n",
    "    ''' remove org.apache. like texts'''\n",
    "    d =  re.sub('(\\w+\\.){2,}','',d)\n",
    "\n",
    "    '''Remove additional white spaces'''\n",
    "    d = re.sub('[\\s]+', ' ', d)\n",
    "    \n",
    "    d = re.sub('\\.(?!$)', '', d)\n",
    "\n",
    "    '''Replace #word with word'''\n",
    "    d = re.sub(r'#([^\\s]+)', r'\\1', d)\n",
    "\n",
    "    return d "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeNonenglishCharac(s):\n",
    "    return re.sub('\\W+','', s )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "extra_punc = ['','.', '``', '...', '\\'s', '--', '-', 'n\\'t', '_', '–','&']\n",
    "\n",
    "sw_list = stopwords.words('english') + list(string.punctuation)+ extra_punc + ['u','the','us','say','that',\n",
    "                     'he','me','she','get','rt','it','mt','via','not','and','let','so','say','dont','use','you']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanText(d):\n",
    "    ''' WordNetLemmatizer'''\n",
    "    wordnet_lemmatizer = WordNetLemmatizer()\n",
    "    \n",
    "    ''' Stemming '''\n",
    "    #stem = Porterstem() \n",
    "    stem=PorterStemmer()\n",
    "    tokenizer=TweetTokenizer()\n",
    "    d = unidecode(d)\n",
    "    d = expand_contractions(d)\n",
    "    tokens = tokenizer.tokenize(d)\n",
    "    d = ' '.join([tok for tok in tokens if len(tok) > 2 if tok not in sw_list and not tok.isdigit()])\n",
    "    d = re.sub('\\b\\w{,2}\\b', '', d)\n",
    "    d = re.sub(' +', ' ', d)\n",
    "    \n",
    "    ''' calling functions '''\n",
    "    d = remove_special_chars(d)\n",
    "    d = removeEmoji(d)\n",
    "    d= [stem.stem(w) for w in d.split()]\n",
    "    return ' '.join([wordnet_lemmatizer.lemmatize(word) for word in d])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['text'] = train['text'].apply(lambda x: cleanText(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['label']\n",
    "\n",
    "''' train val split '''\n",
    "X_train, X_val, y_train, y_val = train_test_split(train['text'], y_train, test_size=0.2, random_state=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TPU is not initialized \n"
     ]
    }
   ],
   "source": [
    "try:    \n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    print('TPU is not initialized ')\n",
    "    tpu = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Replicas num:  1\n"
     ]
    }
   ],
   "source": [
    "if tpu:\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "    \n",
    "print(\"Replicas num: \", strategy.num_replicas_in_sync)\n",
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "'''Configuration of hyperparameters'''\n",
    "epochs = 3\n",
    "BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n",
    "max_len = 192"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Tokenize the data and separate them in chunks of 256 units'''\n",
    "\n",
    "max_len=512\n",
    "chunk_size=256\n",
    "\n",
    "def quickEnc(t,token, maxlen=max_len):\n",
    "    encoder_di = token.batch_encode_plus(t, return_attention_mask=False, return_token_type_ids=False, \n",
    "                                         pad_to_max_length=True, max_length=max_len, truncation=True)\n",
    "    \n",
    "    ''' converting into array '''\n",
    "    return np.array(encoder_di[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Model function '''\n",
    "\n",
    "def Model(transformer, X_train, X_val, batch_size, img_name, max_len=512):\n",
    "    \n",
    "    ''' input layer '''\n",
    "    inp_words_ids = Input(shape =(max_len,),dtype = tf.int32,name=\"input_word_ids\")\n",
    "    seq_out = transformer(inp_words_ids)[0]\n",
    "    cls_t = seq_out[:,0,:]\n",
    "    output =  Dense(3,activation='softmax')(cls_t)\n",
    "    \n",
    "    ''' Model '''\n",
    "    model = Model(inputs =inp_words_ids,outputs=output)\n",
    "    \n",
    "    ''' compile the model '''\n",
    "    model.compile(Adam(lr=1e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    ''' lets see how model looks like '''\n",
    "    model.summary()\n",
    "    \n",
    "    plot_model(model,to_file=img_name, show_shapes=True, show_layer_names=True, rankdir=\"TB\", expand_nested=False, \n",
    "               dpi=96)\n",
    "    \n",
    "    ''' training '''\n",
    "    model.fit(X_train, steps_per_epoch=BATCH_SIZE, validation_data=X_val, epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delObects(*args):\n",
    "    for arg in args:\n",
    "        del arg\n",
    "        gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at distilbert-base-multilingual-cased were not used when initializing TFDistilBertModel: ['vocab_projector', 'vocab_transform', 'activation_13', 'vocab_layer_norm']\n",
      "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-multilingual-cased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Model() got an unexpected keyword argument 'inputs'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_17864/2290269806.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     27\u001b[0m     \u001b[1;34m''' Model '''\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m     Model(transf_layer, train_ds, val_ds, X_train_enc.shape[0], \"Distilbert_Multilingual_Transformer.png\", \n\u001b[1;32m---> 29\u001b[1;33m           max_len=max_len)\n\u001b[0m\u001b[0;32m     30\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m     \u001b[0mdelObects\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_enc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_val_enc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_ds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_ds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdist_token\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransf_layer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_17864/3402411008.py\u001b[0m in \u001b[0;36mModel\u001b[1;34m(transformer, X_train, X_val, batch_size, img_name, max_len)\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;34m''' Model '''\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0minp_words_ids\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[1;34m''' compile the model '''\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Model() got an unexpected keyword argument 'inputs'"
     ]
    }
   ],
   "source": [
    "''' BERT '''\n",
    "\n",
    "with strategy.scope():\n",
    "\n",
    "    dist_token = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n",
    "    X_train_enc= quickEnc(X_train.astype(str),dist_token, maxlen=max_len)\n",
    "    X_val_ebc = quickEnc(X_val.astype(str), dist_token, maxlen=max_len)\n",
    "    \n",
    "    ''' input pipe line '''\n",
    "    train_ds = (\n",
    "        tf.data.Dataset.from_tensor_slices((X_train_enc, y_train))\n",
    "        .repeat()\n",
    "        .shuffle(2048)\n",
    "        .batch(BATCH_SIZE)\n",
    "        .prefetch(AUTO)\n",
    "    )\n",
    "    \n",
    "    val_ds = (\n",
    "        tf.data.Dataset.from_tensor_slices((X_val_ebc, y_val))\n",
    "        .batch(BATCH_SIZE)\n",
    "        .cache()\n",
    "        .prefetch(AUTO)                \n",
    "    )\n",
    "    \n",
    "    transf_layer = TFAutoModel.from_pretrained('distilbert-base-multilingual-cased')\n",
    "    \n",
    "    ''' Model '''\n",
    "    Model(transf_layer, train_ds, val_ds, X_train_enc.shape[0], \"Distilbert_Multilingual_Transformer.png\", \n",
    "          max_len=max_len)\n",
    "    \n",
    "    delObects(X_train_enc, X_val_enc, train_ds, val_ds, dist_token, transf_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (Temp/ipykernel_17864/3154460974.py, line 17)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"C:\\Users\\KAIF\\AppData\\Local\\Temp/ipykernel_17864/3154460974.py\"\u001b[1;36m, line \u001b[1;32m17\u001b[0m\n\u001b[1;33m    val_ds = (\u001b[0m\n\u001b[1;37m         ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "''' Albert Transformer '''\n",
    "\n",
    "with strategy.scope():\n",
    "\n",
    "    alb_token = transformers.AlbertTokenizer.from_pretrained('albert-base-v1')\n",
    "    X_train_enc= quickEnc(X_train.astype(str), alb_token, maxlen=max_len)\n",
    "    X_val_enc = quickEnc(X_val.astype(str), alb_token, maxlen=max_len)\n",
    "    \n",
    "    ''' input pipe line '''\n",
    "    train_ds = (\n",
    "                    tf.data.Dataset.from_tensor_slices((X_train_enc, y_train)\n",
    "                    .repeat()\n",
    "                    .shuffle(2048)\n",
    "                    .batch(BATCH_SIZE)\n",
    "                    .prefetch(AUTO)\n",
    "                    )\n",
    "    val_ds = (\n",
    "        tf.data.Dataset.from_tensor_slices((X_val_enc, y_val))\n",
    "        .batch(BATCH_SIZE)\n",
    "        .cache()\n",
    "        .prefetch(AUTO)                \n",
    "    )\n",
    "        \n",
    "    transf_layer = TFAutoModel.from_pretrained('albert-base-v1')\n",
    "    \n",
    "    Model(transf_layer, train_ds, val_ds, X_train_enc.shape[0], \"AlbertTransformer.png\", max_len=max+len)\n",
    "    \n",
    "    delObects(X_train_enc, X_val_enc, train_ds, val_ds, alb_token, transf_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' XLM Roberta/Roberta '''\n",
    "\n",
    "with strategy.scope():\n",
    "\n",
    "    xlm_roberta_token = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')\n",
    "    X_train_enc = quickEnc(X_train.astype(str), xlm_roberta_token, maxlen=max_len)\n",
    "    X_val_enc = quickEnc(X_val.astype(str), xlm_roberta_token, maxlen=max_len)\n",
    "    \n",
    "    ''' input pipe line '''\n",
    "    train_ds = (\n",
    "                tf.data.Dataset.from_tensor_slices((X_train_enc, y_train))\n",
    "                .repeat()\n",
    "                .shuffle(2048)\n",
    "                .batch(BATCH_SIZE)\n",
    "                .prefetch(AUTO)\n",
    "                )\n",
    "    val_ds = (\n",
    "        tf.data.Dataset.from_tensor_slices((X_val_enc, y_val))\n",
    "        .batch(BATCH_SIZE)\n",
    "        .cache()\n",
    "        .prefetch(AUTO)                \n",
    "    )\n",
    "    \n",
    "    transf_layer = TFAutoModel.from_pretrained('xlm-roberta-base')\n",
    "    \n",
    "    Model(transf_layer, train_ds, val_ds, X_train_enc.shape[0], \"Roberta-Transformer.png\",max_len=max_len)\n",
    "    \n",
    "    delObects(X_train_enc, X_val_enc, train_ds, val_ds, xlm_roberta_token, transf_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' GPT-Generative Pretraining '''\n",
    "\n",
    "with strategy.scope():\n",
    "\n",
    "    gpt2_token = transformers.GPT2Tokenizer.from_pretrained('gpt2-medium')\n",
    "    X_train_enc= quickEnc(X_train.astype(str),gpt2_token, maxlen=max_len)\n",
    "    X_val_enc = quickEnc(X_val.astype(str), gpt2_token, maxlen=max_len)\n",
    "    \n",
    "    ''' input pipe line '''\n",
    "    train_ds = (\n",
    "                tf.data.Dataset.from_tensor_slices((X_train_enc, y_train))\n",
    "                .repeat()\n",
    "                .shuffle(2048)\n",
    "                .batch(BATCH_SIZE)\n",
    "                .prefetch(AUTO)\n",
    "                )\n",
    "    val_ds = (\n",
    "        tf.data.Dataset.from_tensor_slices((X_val_enc, y_val))\n",
    "        .batch(BATCH_SIZE)\n",
    "        .cache()\n",
    "        .prefetch(AUTO)                \n",
    "    )\n",
    "    \n",
    "    transf_layer = TFAutoModel.from_pretrained('gpt2-medium')\n",
    "    \n",
    "    Model(transf_layer, train_ds, val_ds, X_train_enc.shape[0], \"GPT2-Transformer.png\", max_len=max_len)\n",
    "    \n",
    "    delObects(X_train_enc, X_val_enc, train_ds, val_ds, gpt2_token,transf_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "OpenCV_Project",
   "language": "python",
   "name": "opencv_project"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}