main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install google\n",
    "!pip install beautifulsoup4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googlesearch import search\n",
    "import requests\n",
    "import re\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "async def scraper(url):\n",
    "    #  Scrapy Selenium\n",
    "    def removeElement(soup,element: str):\n",
    "        elements = soup.find_all(element)\n",
    "        if len(elements)==0:\n",
    "            return soup\n",
    "        for div in elements:\n",
    "            div.decompose()\n",
    "        return soup\n",
    "\n",
    "    def removeElementsWithClass(soup,class_:str):\n",
    "        class_regex = re.compile(rf\"{class_}\")\n",
    "        elements = soup.find_all(class_=class_regex)\n",
    "        if len(elements)>0:\n",
    "            for tag in elements:\n",
    "                tag.decompose()\n",
    "        element = soup.find(id=class_regex)\n",
    "        if element:\n",
    "            element.decompose()\n",
    "        return soup\n",
    "\n",
    "    response = requests.get(url)\n",
    "    if response.status_code != 200:\n",
    "        print(\"Failed to retrieve webpage\")\n",
    "        return\n",
    "\n",
    "    soup = BeautifulSoup(response.content, 'html.parser').find('body')\n",
    "    soup = removeElement(soup,'nav')\n",
    "    soup = removeElement(soup,'footer')\n",
    "    soup = removeElementsWithClass(soup,'menu')\n",
    "    soup = removeElementsWithClass(soup,'nav')\n",
    "    soup = removeElementsWithClass(soup,'footer')\n",
    "    # for heading in soup.find_all(['h1','h2']):\n",
    "    #     print(heading)\n",
    "    # for para in soup.find_all('p'):\n",
    "    #     print(para)\n",
    "    return soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter, Language\n",
    "from langchain.document_loaders import TextLoader\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain_openai import OpenAIEmbeddings, ChatOpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createChunking(website):\n",
    "    splitter = RecursiveCharacterTextSplitter.from_language(chunk_size=1024, chunk_overlap=30,language=Language.PYTHON)\n",
    "    chunks = splitter.split_documents(website)\n",
    "    print(chunks)\n",
    "    return chunks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_k_websites = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = input('Enter your query')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.elastic.co/what-is/large-language-models\n",
      "https://www.elastic.co/what-is/natural-language-processing\n"
     ]
    }
   ],
   "source": [
    "\n",
    "response = search(query=query,num=top_k_websites,stop=top_k_websites)\n",
    "pages = []\n",
    "for i, result in enumerate(response, start=1):\n",
    "    pages.append(await scraper(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = 'scraped.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(DATA_PATH,'w') as file:\n",
    "    for page in pages:\n",
    "        file.write(page.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "API_KEY = os.environ['OPENAI_API_KEY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = TextLoader(DATA_PATH).load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Skip to main contentWhat is a large language model (LLM)?Explore popular open-source LLMsLarge language model definitionA large language model (LLM) is a deep learning algorithm that can perform a variety of natural language processing (NLP) tasks. Large language models use transformer models and are trained using massive datasets — hence, large. This enables them to recognize, translate, predict, or generate text or other content.Large language models are also referred to as neural networks (NNs), which are computing systems inspired by the human brain. These neural networks work using a network of nodes that are layered, much like neurons.In addition to teaching human languages to artificial intelligence (AI) applications, large language models can also be trained to perform a variety of tasks like understanding protein structures, writing software code, and more. Like the human brain, large language models must be pre-trained and then fine-tuned so that they can solve text classification, question answering, document summarization, and text generation problems. Their problem-solving capabilities can be applied to fields like healthcare, finance, and entertainment where large language models serve a variety of NLP applications, such as translation, chatbots, AI assistants, and so on.Large language models also have large numbers of parameters, which are akin to memories the model collects as it learns from training. Think of these parameters as the model’s knowledge bank.So, what is a transformer model? A transformer model is the most common architecture of a large language model. It consists of an encoder and a decoder. A transformer model processes data by tokenizing the input, then simultaneously conducting mathematical equations to discover relationships between tokens. This enables the computer to see the patterns a human would see were it given the same query.Transformer models work with self-attention mechanisms, which enables the model to learn more quickly than traditional models like long short-term memory models. Self-attention is what enables the transformer model to consider different parts of the sequence, or the entire context of a sentence, to generate predictions.Related: Apply transformers to your search applicationsKey components of large language modelsLarge language models are composed of multiple neural network layers. Recurrent layers, feedforward layers, embedding layers, and attention layers work in tandem to process the input text and generate output content.The embedding layer creates embeddings from the input text. This part of the large language model captures the semantic and syntactic meaning of the input, so the model can understand context.The feedforward layer (FFN) of a large language model is made of up multiple fully connected layers that transform the input embeddings. In so doing, these layers enable the model to glean higher-level abstractions — that is, to understand the user\\'s intent with the text input.The recurrent layer interprets the words in the input text in sequence. It captures the relationship between words in a sentence.The attention mechanism enables a language model to focus on single parts of the input text that is relevant to the task at hand. This layer allows the model to generate the most accurate outputs.There are three main kinds of large language models:Generic or raw language models predict the next word based on the language in the training data. These language models perform information retrieval tasks.Instruction-tuned language models are trained to predict responses to the instructions given in the input. This allows them to perform sentiment analysis, or to generate text or code.Dialog-tuned language models are trained to have a dialog by predicting the next response. Think of chatbots or conversational AI.What is the difference between large language models and generative AI?Generative AI is an umbrella term that refers to artificial intelligence models that have the capability to generate content. Generative AI can generate text, code, images, video, and music. Examples of generative AI include Midjourney, DALL-E, and ChatGPT.Large language models are a type of generative AI that are trained on text and produce textual content. ChatGPT is a popular example of generative text AI.All large language models are generative AI1.How do large language models work?A large language model is based on a transformer model and works by receiving an input, encoding it, and then decoding it to produce an output prediction. But before a large language model can receive text input and generate an output prediction, it requires training, so that it can fulfill general functions, and fine-tuning, which enables it to perform specific tasks.Training: Large language models are pre-trained using large textual datasets from sites like Wikipedia, GitHub, or others. These datasets consist of trillions of words, and their quality will affect the language model\\'s performance. At this stage, the large language model engages in unsupervised learning, meaning it processes the datasets fed to it without specific instructions. During this process, the LLM\\'s AI algorithm can learn the meaning of words, and of the relationships between words. It also learns to distinguish words based on context. For example, it would learn to understand whether \"right\" means \"correct,\" or the opposite of \"left', metadata={'source': 'scraped.txt'})]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedder = OpenAIEmbeddings(api_key=API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks = createChunking(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = Chroma.from_documents(chunks, embedder,persist_directory='./database')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "db.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = Chroma(persist_directory=DATA_PATH,embedding_function=embedder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "context = db.similarity_search_with_relevance_scores(query,k=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(Document(page_content='Skip to main contentWhat is a large language model (LLM)?Explore popular open-source LLMsLarge language model definitionA large language model (LLM) is a deep learning algorithm that can perform a variety of natural language processing (NLP) tasks. Large language models use transformer models and are trained using massive datasets — hence, large. This enables them to recognize, translate, predict, or generate text or other content.Large language models are also referred to as neural networks (NNs), which are computing systems inspired by the human brain. These neural networks work using a network of nodes that are layered, much like neurons.In addition to teaching human languages to artificial intelligence (AI) applications, large language models can also be trained to perform a variety of tasks like understanding protein structures, writing software code, and more. Like the human brain, large language models must be pre-trained and then fine-tuned so that they can solve text classification, question', metadata={'source': 'scraped.txt'}), 0.8211523777245459), (Document(page_content='text classification, question answering, document summarization, and text generation problems. Their problem-solving capabilities can be applied to fields like healthcare, finance, and entertainment where large language models serve a variety of NLP applications, such as translation, chatbots, AI assistants, and so on.Large language models also have large numbers of parameters, which are akin to memories the model collects as it learns from training. Think of these parameters as the model’s knowledge bank.So, what is a transformer model? A transformer model is the most common architecture of a large language model. It consists of an encoder and a decoder. A transformer model processes data by tokenizing the input, then simultaneously conducting mathematical equations to discover relationships between tokens. This enables the computer to see the patterns a human would see were it given the same query.Transformer models work with self-attention mechanisms, which enables the model to learn more quickly than', metadata={'source': 'scraped.txt'}), 0.7714181820213781), (Document(page_content='to artificial intelligence models that have the capability to generate content. Generative AI can generate text, code, images, video, and music. Examples of generative AI include Midjourney, DALL-E, and ChatGPT.Large language models are a type of generative AI that are trained on text and produce textual content. ChatGPT is a popular example of generative text AI.All large language models are generative AI1.How do large language models work?A large language model is based on a transformer model and works by receiving an input, encoding it, and then decoding it to produce an output prediction. But before a large language model can receive text input and generate an output prediction, it requires training, so that it can fulfill general functions, and fine-tuning, which enables it to perform specific tasks.Training: Large language models are pre-trained using large textual datasets from sites like Wikipedia, GitHub, or others. These datasets consist of trillions of words, and their quality will affect the', metadata={'source': 'scraped.txt'}), 0.7623094323313777), (Document(page_content='their quality will affect the language model\\'s performance. At this stage, the large language model engages in unsupervised learning, meaning it processes the datasets fed to it without specific instructions. During this process, the LLM\\'s AI algorithm can learn the meaning of words, and of the relationships between words. It also learns to distinguish words based on context. For example, it would learn to understand whether \"right\" means \"correct,\" or the opposite of \"left', metadata={'source': 'scraped.txt'}), 0.7468296416864612), (Document(page_content=\"user's intent with the text input.The recurrent layer interprets the words in the input text in sequence. It captures the relationship between words in a sentence.The attention mechanism enables a language model to focus on single parts of the input text that is relevant to the task at hand. This layer allows the model to generate the most accurate outputs.There are three main kinds of large language models:Generic or raw language models predict the next word based on the language in the training data. These language models perform information retrieval tasks.Instruction-tuned language models are trained to predict responses to the instructions given in the input. This allows them to perform sentiment analysis, or to generate text or code.Dialog-tuned language models are trained to have a dialog by predicting the next response. Think of chatbots or conversational AI.What is the difference between large language models and generative AI?Generative AI is an umbrella term that refers to artificial intelligence\", metadata={'source': 'scraped.txt'}), 0.7287668130767889)]\n"
     ]
    }
   ],
   "source": [
    "print(context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import ChatPromptTemplate\n",
    "prompt_template = ChatPromptTemplate.from_template(\n",
    "\"\"\"\n",
    "Answer the question based only on the following context:\n",
    "{context}\n",
    "\n",
    "Answer the question based only on the above context: {query}\n",
    "\"\"\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = prompt_template.format(context=context,query=query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ChatOpenAI(api_key=API_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/parteek/.local/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `predict` was deprecated in LangChain 0.1.7 and will be removed in 0.2.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "result = model.predict(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A large language model (LLM) is a deep learning algorithm that can perform a variety of natural language processing (NLP) tasks. Large language models use transformer models and are trained using massive datasets, enabling them to recognize, translate, predict, or generate text or other content. They are also referred to as neural networks (NNs) and can be trained to perform tasks beyond language processing, such as understanding protein structures or writing software code.\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}