Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a96744e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: BeautifulSoup4 in c:\\users\\kaif\\anaconda3\\lib\\site-packages (4.9.3)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\kaif\\anaconda3\\lib\\site-packages (from BeautifulSoup4) (2.2.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.3.1; however, version 22.0.3 is available.\n",
      "You should consider upgrading via the 'c:\\users\\kaif\\anaconda3\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pydot in c:\\users\\kaif\\anaconda3\\lib\\site-packages (1.4.2)\n",
      "Requirement already satisfied: pyparsing>=2.1.4 in c:\\users\\kaif\\anaconda3\\lib\\site-packages (from pydot) (2.4.7)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.3.1; however, version 22.0.3 is available.\n",
      "You should consider upgrading via the 'c:\\users\\kaif\\anaconda3\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install BeautifulSoup4\n",
    "!pip install pydot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ce6c7982",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import Counter,defaultdict\n",
    "import gc\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "import matplotlib\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.stem import WordNetLemmatizer,PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "import re\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score,StratifiedKFold,KFold,StratifiedShuffleSplit,cross_val_predict\n",
    "from lightgbm import LGBMClassifier as lg\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.layers import LSTM, Dense,Flatten,Conv2D,Conv1D,GlobalMaxPooling1D,GlobalMaxPool1D\n",
    "#from keras.optimizers import Adam\n",
    "import numpy as np  \n",
    "import pandas as pd \n",
    "import keras.backend as k\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed, Bidirectional,GRU\n",
    "from tensorflow.keras.models import Model,Sequential\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "#from kaggle_datasets import KaggleDatasets\n",
    "import transformers\n",
    "from tqdm.notebook import tqdm\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from transformers import AutoTokenizer, pipeline, TFDistilBertModel,TFAutoModel\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "import string\n",
    "from unidecode import unidecode\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "12e3455f",
   "metadata": {},
   "outputs": [],
   "source": [
    "''' reading files '''\n",
    "train_df = pd.read_json(\"D:/NLP Projects 33 - GitHub Bugs Prediction/archive/embold_train.json\").reset_index(drop=True)\n",
    "test_df = pd.read_json(\"D:/NLP Projects 33 - GitHub Bugs Prediction/archive/embold_test.json\").reset_index(drop=True)\n",
    "train_extra= pd.read_json(\"D:/NLP Projects 33 - GitHub Bugs Prediction/archive/embold_train_extra.json\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0e2140e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fx(x):\n",
    "    return x['title'] + \" \" + x['body']   \n",
    "train_df['text']= train_df.apply(lambda x : fx(x),axis=1)\n",
    "test_df['text']= test_df.apply(lambda x : fx(x),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce52ddf9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "OpenCV_Project",
   "language": "python",
   "name": "opencv_project"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}