04_Ragas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import DirectoryLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "app_dir = os.path.join(os.getcwd(), \"app\")\n",
    "load_dotenv(os.path.join(app_dir, \".env\"))\n",
    "\n",
    "loader = DirectoryLoader(\"./data\", glob=\"**/*.txt\")\n",
    "docs = loader.load()\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=350,\n",
    "    chunk_overlap=20,\n",
    "    length_function=len,\n",
    "    is_separator_regex=False,\n",
    ")\n",
    "chunks = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RAGAS expects a file_name dict as key\n",
    "for document in chunks:\n",
    "    document.metadata[\"file_name\"] = document.metadata[\"source\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ragas.testset.generator import TestsetGenerator\n",
    "from ragas.testset.evolutions import simple, reasoning, multi_context\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "embeddings = OpenAIEmbeddings()\n",
    "model = ChatOpenAI()\n",
    "\n",
    "generator = TestsetGenerator.from_langchain(\n",
    "    embeddings=embeddings, generator_llm=model, critic_llm=model\n",
    ")\n",
    "\n",
    "testset = generator.generate_with_langchain_docs(\n",
    "    chunks,\n",
    "    test_size=8,\n",
    "    distributions={simple: 0.5, reasoning: 0.25, multi_context: 0.25},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testset.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai.embeddings import OpenAIEmbeddings\n",
    "\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "embedding = OpenAIEmbeddings()\n",
    "model = ChatOpenAI()\n",
    "\n",
    "vectorstore = Chroma.from_documents(chunks, embedding)\n",
    "retriever = vectorstore.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.prompts import PromptTemplate\n",
    "\n",
    "template = \"\"\"Answer the question based only on the following context:\n",
    "{context}\n",
    "\n",
    "Question: {question}\n",
    "\"\"\"\n",
    "\n",
    "prompt = PromptTemplate(template=template, input_variables=[\"context\", \"question\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.schema.runnable import RunnablePassthrough\n",
    "from langchain.schema.output_parser import StrOutputParser\n",
    "\n",
    "rag_chain = (\n",
    "    {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
    "    | prompt\n",
    "    | model\n",
    "    | StrOutputParser()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# questions = testset.to_pandas()[\"question\"].to_list()\n",
    "# ground_truth = testset.to_pandas()[\"ground_truth\"].to_list()\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"./questions_answers/qa.csv\", delimiter=\";\")\n",
    "questions = df[\"question\"].tolist()\n",
    "ground_truth = df[\"ground_truth\"].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "data = {\"question\": [], \"answer\": [], \"contexts\": [], \"ground_truth\": ground_truth}\n",
    "\n",
    "for query in questions:\n",
    "    data[\"question\"].append(query)\n",
    "    data[\"answer\"].append(rag_chain.invoke(query))\n",
    "    data[\"contexts\"].append(\n",
    "        [doc.page_content for doc in retriever.invoke(query)]\n",
    "    )\n",
    "\n",
    "dataset = Dataset.from_dict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_entry = {\n",
    "    \"question\": data[\"question\"][0],\n",
    "    \"answer\": data[\"answer\"][0],\n",
    "    \"contexts\": data[\"contexts\"][0],\n",
    "    \"ground_truth\": data[\"ground_truth\"][0],\n",
    "}\n",
    "first_entry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ragas import evaluate\n",
    "from ragas.metrics import (\n",
    "    faithfulness,\n",
    "    answer_relevancy,\n",
    "    context_relevancy,\n",
    "    context_recall,\n",
    "    context_precision,\n",
    ")\n",
    "\n",
    "result = evaluate(\n",
    "    dataset=dataset,\n",
    "    metrics=[\n",
    "        context_relevancy,\n",
    "        context_precision,\n",
    "        context_recall,\n",
    "        faithfulness,\n",
    "        answer_relevancy,\n",
    "    ],\n",
    "    llm=model,\n",
    "    embeddings=embedding\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import LinearSegmentedColormap\n",
    "\n",
    "df = result.to_pandas()\n",
    "\n",
    "heatmap_data = df[\n",
    "    [\n",
    "        \"context_relevancy\",\n",
    "        \"context_precision\",\n",
    "        \"context_recall\",\n",
    "        \"faithfulness\",\n",
    "        \"answer_relevancy\",\n",
    "    ]\n",
    "]\n",
    "\n",
    "cmap = LinearSegmentedColormap.from_list(\"green_red\", [\"red\", \"green\"])\n",
    "\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(heatmap_data, annot=True, fmt=\".2f\", linewidths=0.5, cmap=cmap)\n",
    "\n",
    "plt.yticks(ticks=range(len(df[\"question\"])), labels=df[\"question\"], rotation=0)\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "app",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}