finishing QnA and functions calling plus pydantic

notebooks/persisted-embedding-model.ipynb → archive/notebooks/002_persisted-embedding-model.ipynb

@@ -8,14 +8,14 @@
    "outputs": [],
    "source": [
     "import openai\n",
-    "from llama_index import SimpleDirectoryReader\n",
-    "from llama_index import Document\n",
-    "from llama_index import VectorStoreIndex\n",
-    "from llama_index import ServiceContext\n",
-    "from llama_index.llms import OpenAI\n",
+    "from llama_index.core import SimpleDirectoryReader\n",
+    "from llama_index.core import Document\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.core import ServiceContext\n",
+    "from llama_index.llms.openai import OpenAI\n",
     "\n",
-    "from llama_index.embeddings import HuggingFaceEmbedding\n",
-    "from llama_index import StorageContext, load_index_from_storage\n",
+    "from llama_index.legacy.embeddings import HuggingFaceEmbedding\n",
+    "from llama_index.core import StorageContext, load_index_from_storage\n",
     "\n",
     "import time"
    ]
@@ -298,12 +298,123 @@
    "outputs": [],
    "source": []
   },
+  {
+   "cell_type": "markdown",
+   "id": "4730d2e7-dfa9-452d-ae20-95a24b6fd8bd",
+   "metadata": {},
+   "source": [
+    "### Create vectorstore based on default storage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67e3e5ea-8b6d-4493-91ca-f0ced02971e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core import (\n",
+    "    SimpleDirectoryReader,\n",
+    "    VectorStoreIndex,\n",
+    "    StorageContext,\n",
+    "    load_index_from_storage,\n",
+    ")\n",
+    "\n",
+    "from llama_index.core.tools import QueryEngineTool, ToolMetadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d8d4ee04-37be-4014-8403-27ef467462b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "documents = SimpleDirectoryReader(input_files=[\n",
+    "                                    \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "                                    \"../raw_documents/qna.txt\"\n",
+    "                                  ]).load_data()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d5f6a8b0-0798-42ab-9741-caf52a0bae0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_index = VectorStoreIndex.from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e9f4da3-32af-4e1a-9b28-bfe2c51cc7e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_index.storage_context.persist(persist_dir=\"../models/default_storage\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f032e21-7f98-4752-ad55-3607fd40b7d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_engine = hi_index.as_query_engine(similarity_top_k=3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c599039-8cdf-449c-b3d8-f0d41bffdffc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = hi_engine.query(\"what is the healthcare philosophy in singapore\")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0757962c-235b-49b7-a92d-49c474b79731",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b77caa5b-16eb-48ec-8f74-0fc6ca03efbd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f0c97e94-5dc5-443e-94a2-ce28aad99a3c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c19cc1b2-4669-4019-9fea-8e8aa3c74c6f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "id": "8acae3ed-2953-45a3-aba9-0327b6ae3679",
    "metadata": {},
    "source": [
-    "### ChromaDB method - create vectorstore"
+    "### ChromaDB method - create vectorstore based on Chroma"
    ]
   },
   {
@@ -314,13 +425,19 @@
    "outputs": [],
    "source": [
     "import chromadb\n",
-    "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
-    "from llama_index.vector_stores import ChromaVectorStore\n",
-    "from llama_index.storage.storage_context import StorageContext\n",
-    "from llama_index import ServiceContext\n",
-    "from llama_index import Document\n",
+    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.core import ServiceContext\n",
+    "from llama_index.core import Document\n",
+    "\n",
+    "from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()\n",
     "\n",
-    "from llama_index.embeddings import HuggingFaceEmbedding\n",
     "\n",
     "import time"
    ]
@@ -333,7 +450,11 @@
    "outputs": [],
    "source": [
     "# load some documents\n",
-    "documents = SimpleDirectoryReader(input_files=[\"../raw_documents/HI_Knowledge_Base.pdf\"]).load_data()\n",
+    "documents = SimpleDirectoryReader(input_files=[\n",
+    "                                    \"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "                                    \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "                                    \"../raw_documents/qna.txt\"\n",
+    "                                  ]).load_data()\n",
     "document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))"
    ]
   },
@@ -385,19 +506,40 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")"
+    "# embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "# embed_model = HuggingFaceEmbedding(model_name=\"local:../models/fine-tuned-embeddings\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "5289f0f9-bce2-4a2f-9428-b99906b78622",
+   "id": "0946b6ce-96ab-44de-ad75-e424a8429f67",
    "metadata": {},
    "outputs": [],
    "source": [
-    "service_context = ServiceContext.from_defaults(llm=None, embed_model=embed_model)"
+    "Settings.llm = None\n",
+    "Settings.chunk_size = 1024\n",
+    "Settings.embed_model = \"local:../models/fine-tuned-embeddings\""
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b8c73a2c-1129-406a-8046-085afcaf9cbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nodes = Settings.node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adfe688f-95c0-477c-a9de-e9e77541a1d7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -408,6 +550,24 @@
     "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6a764113-ad7e-4674-aa57-ebbf405902a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context.docstore.add_documents(nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38e7c88d-6c45-4275-8293-d09b4b85a7cf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -418,6 +578,67 @@
     "start_time = time.time()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cbd11b89-9b83-4f08-bb30-160f750f2ffb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_index = VectorStoreIndex(nodes, storage_context=storage_context)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "082a0d7e-b025-4db1-be2a-7a0b7bc453b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_query_engine = vector_index.as_query_engine()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d3bd848d-9985-4a3d-bdc4-ec340cc69ef3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "indexing_cost = time.time() - start_time\n",
+    "indexing_cost = indexing_cost / 60\n",
+    "print(f\"Indexing time: {indexing_cost:.1f} mins\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3290e870-41d7-49c4-9c4f-cb16bd1f469e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = vector_query_engine.query(\"what is the healthcare philosophy in singapore\")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "131d907a-0677-4ad8-b3f7-6fc9b9c5d0a5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f95e42ff-374b-4c57-8333-137a6205a6ad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start_time = time.time()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -426,6 +647,8 @@
    "outputs": [],
    "source": [
     "# create your index\n",
+    "service_context = ServiceContext.from_defaults(llm=None, embed_model=\"local:../models/fine-tuned-embeddings\")\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
     "index = VectorStoreIndex.from_documents(\n",
     "    documents, service_context=service_context, storage_context=storage_context\n",
     ")"
@@ -471,7 +694,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r_list = retriever.retrieve(\"What is the meaning of life?\")"
+    "r_list = retriever.retrieve(\"what is the healthcare philosophy in singapore\")"
    ]
   },
   {
@@ -493,7 +716,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "response = query_engine.query(\"What is the meaning of life?\")"
+    "response = query_engine.query(\"what is the healthcare philosophy in singapore\")"
    ]
   },
   {
@@ -519,7 +742,7 @@
    "id": "a7fc01f6-4738-415b-a96b-afd6cf8d789a",
    "metadata": {},
    "source": [
-    "### ChromaDB method - load vectorstore"
+    "### ChromaDB method - load vectorstore based on Chroma"
    ]
   },
   {

archive/notebooks/005_qna_prompting_with_pydantic_embeddings.ipynb

@@ -0,0 +1,528 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4cf3ca7c-2c43-495b-a1ee-24c770f0ad1e",
+   "metadata": {},
+   "source": [
+    "### Simple OpenAI agent with tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a9d74b11-4049-4e3c-839e-7d13d7c0dadc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "from typing import Sequence, List\n",
+    "from pydantic import BaseModel\n",
+    "\n",
+    "from llama_index.core.llms import ChatMessage\n",
+    "from llama_index.core.tools import BaseTool, FunctionTool\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.agent.openai import OpenAIAgent\n",
+    "\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39ca840b-0ec7-4279-9654-e4ddfda6137f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define sample Tool\n",
+    "def multiply(a: int, b: int) -> int:\n",
+    "    \"\"\"Multiply two integers and returns the result integer\"\"\"\n",
+    "    return a * b\n",
+    "\n",
+    "multiply_tool = FunctionTool.from_defaults(fn=multiply)\n",
+    "\n",
+    "def add(a: int, b: int) -> int:\n",
+    "    \"\"\"Add two integers and returns the result integer\"\"\"\n",
+    "    return a + b\n",
+    "\n",
+    "add_tool = FunctionTool.from_defaults(fn=add)\n",
+    "\n",
+    "class Song(BaseModel):\n",
+    "    \"\"\"A song with name and artist\"\"\"\n",
+    "\n",
+    "    name: str\n",
+    "    artist: str\n",
+    "\n",
+    "song_fn = FunctionTool.from_defaults(fn=Song)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4df3e9f2-4a32-4449-b203-929dff9e7963",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbeeea36-0bb0-4edb-9b8c-adb7c64c4cd3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# initialize openai agent\n",
+    "llm = OpenAI(model=\"gpt-3.5-turbo-0613\")\n",
+    "agent = OpenAIAgent.from_tools([multiply_tool, add_tool, song_fn], llm=llm, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65158ede-b99f-477d-9d17-3be40e57a629",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"3 x 2 equals?\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d74b0e5c-47a2-4de4-acd2-d39a94941f2d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9787ed4-46a2-46aa-80e6-b317d9280b9e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"3 plus 2 equals?\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dd3358c6-e0e4-4354-8a4e-97d70254f648",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16e1db2c-dca3-4dc6-9cc5-c10644d5927c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"give me the lyrics of taylor swift's `you belong with me`\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "97a021f4-4701-4914-9ab8-0683b396f096",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1f0b352d-8510-4b2a-a495-9f2e1fbfcddb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# res_stream = agent.stream_chat(message=\"3 x 2 equals?\")\n",
+    "# for r in res_stream.response_gen:\n",
+    "#     print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea0a6cd4-f204-4997-bdfb-cb9b5a9e1266",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "96c978e6-62e2-46e6-ae63-76841487f618",
+   "metadata": {},
+   "source": [
+    "### OpenAI agent with embeddings, and function calling"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f0f5f02-c8e9-43a9-853d-12bb3c19dbe8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import chromadb\n",
+    "\n",
+    "from llama_index.core import (\n",
+    "    SimpleDirectoryReader,\n",
+    "    VectorStoreIndex,\n",
+    "    ServiceContext,\n",
+    "    StorageContext,\n",
+    "    load_index_from_storage,\n",
+    ")\n",
+    "from llama_index.core.memory import ChatMemoryBuffer\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.agent.openai import OpenAIAgent\n",
+    "from llama_index.core.tools import BaseTool, FunctionTool\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "from pydantic import BaseModel\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54855aa4-dcad-404e-843f-c96d61046df3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4edb89f6-bb2f-46ff-8807-dfb03115fcd5",
+   "metadata": {},
+   "source": [
+    "#### Parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "61ad7369-8fd4-434f-b687-0c649940bda1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected_model = \"gpt-3.5-turbo-1106\"\n",
+    "temperature = 0.0\n",
+    "\n",
+    "persisted_vector_db = \"../models/chroma_db\"\n",
+    "input_files = [\"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "               \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "               \"../raw_documents/qna.txt\"]\n",
+    "fine_tuned_path = \"local:../models/fine-tuned-embeddings\"\n",
+    "system_content = (\"You are a helpful study assistant. \"\n",
+    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
+    "                  \"You only respond once as 'Assistant'.\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a18117f5-48a7-4e81-9b46-541f382caf9e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3210c837-9b40-4cd9-bb00-ead559deff6f",
+   "metadata": {},
+   "source": [
+    "#### Load vector store"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9dfba0c-f27d-49d1-86c5-a1d95c11b844",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if os.path.exists(persisted_vector_db):\n",
+    "    db = chromadb.PersistentClient(path=persisted_vector_db)\n",
+    "    chroma_collection = db.get_or_create_collection(\"quickstart\")\n",
+    "    \n",
+    "    # assign chroma as the vector_store to the context\n",
+    "    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "    storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "else:\n",
+    "    documents = SimpleDirectoryReader(input_files=input_files).load_data()\n",
+    "    document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))\n",
+    "    \n",
+    "    # initialize client, setting path to save data\n",
+    "    db = chromadb.PersistentClient(path=persisted_vector_db)\n",
+    "    \n",
+    "    # create collection\n",
+    "    chroma_collection = db.get_or_create_collection(\"quickstart\")\n",
+    "\n",
+    "    # assign chroma as the vector_store to the context\n",
+    "    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "    storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "693c9808-efbe-47a6-a49c-7143c63d13e5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c6bfae0b-7c97-4c2b-9996-f5e3ecf7a992",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define sample Tool\n",
+    "def multiply(a: int, b: int) -> int:\n",
+    "    \"\"\"Multiply two integers and returns the result integer\"\"\"\n",
+    "    return a * b\n",
+    "\n",
+    "def add(a: int, b: int) -> int:\n",
+    "    \"\"\"Add two integers and returns the result integer\"\"\"\n",
+    "    return a + b\n",
+    "\n",
+    "class Song(BaseModel):\n",
+    "    \"\"\"A song with name and artist\"\"\"\n",
+    "\n",
+    "    name: str\n",
+    "    artist: str\n",
+    "\n",
+    "add_tool = FunctionTool.from_defaults(fn=add)\n",
+    "multiply_tool = FunctionTool.from_defaults(fn=multiply)\n",
+    "song_fn = FunctionTool.from_defaults(fn=Song)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16a80b2e-8e5f-462a-8616-042afe18be3a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(model=selected_model, temperature=temperature)\n",
+    "service_context = ServiceContext.from_defaults(llm=llm, embed_model=fine_tuned_path)\n",
+    "\n",
+    "Settings.llm = llm\n",
+    "Settings.embed_model = fine_tuned_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95d3a420-1ee4-45bd-a18b-b398d9531db4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# index = VectorStoreIndex.from_vector_store(\n",
+    "#     vector_store=vector_store,\n",
+    "#     service_context=service_context, \n",
+    "#     storage_context=storage_context\n",
+    "# )\n",
+    "index = VectorStoreIndex.from_vector_store(\n",
+    "    vector_store=vector_store,\n",
+    "    storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eac6d76d-059b-40e3-b67f-c736f1ce6baa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "memory = ChatMemoryBuffer.from_defaults(token_limit=15000)\n",
+    "\n",
+    "hi_engine = index.as_query_engine(\n",
+    "                memory=memory,\n",
+    "                system_prompt=system_content,\n",
+    "                similarity_top_k=3,\n",
+    "                streaming=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18e38a8f-9b51-4675-a1d5-8aaa6c21694c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = hi_engine.query(\"what is the healthcare philosophy in singapore\")\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70dae6f7-682e-42d6-be59-3b807c10482c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1eb5df65-c926-4b22-8071-449d645b339f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_query_tool = QueryEngineTool.from_defaults(\n",
+    "                    query_engine=hi_engine,\n",
+    "                    name=\"vector_tool\",\n",
+    "                    description=(\n",
+    "                        \"Provides information about Health Insurance landscape in Singapore. \"\n",
+    "                        \"Use a detailed plain text question as input to the tool.\"\n",
+    "                    )\n",
+    ")\n",
+    "\n",
+    "# hi_query_tool = QueryEngineTool(\n",
+    "#                     query_engine=hi_engine,\n",
+    "#                     metadata=ToolMetadata(\n",
+    "#                         name=\"health_insurance_mentor\",\n",
+    "#                         description=(\n",
+    "#                             \"Provides information about Health Insurance landscape in Singapore. \"\n",
+    "#                             \"Use a detailed plain text question as input to the tool.\"\n",
+    "#                         )\n",
+    "#                     )\n",
+    "#                 )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5822b1d-32ef-4b68-8629-a727ff51cd0a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2a1235da-a379-4055-8bcf-4b21c91c9fb0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent = OpenAIAgent.from_tools([multiply_tool, add_tool, hi_query_tool], llm=llm, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05b65cbd-d15c-4909-b383-50b13f64e535",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63332a44-9441-4f49-85a2-934e2c55a362",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(\"what is the healthcare philosophy in singapore\", tool_choice=\"auto\")\n",
+    "res.response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81709cbf-9a5e-482f-ae6a-ba361b8219dc",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adf26268-e40a-4ebd-a737-6b203ddc4444",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.stream_chat(\"what is the healthcare philosophy in singapore\", tool_choice=\"auto\")\n",
+    "for r in res.response_gen:\n",
+    "    print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "540c0f71-048a-4a64-9818-e2b1cffc0db7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fbeabf28-30f9-4d7f-a4b9-21cd08a9b128",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.stream_chat(\"what is the result of 328123 + 2891230\", tool_choice=\"auto\")\n",
+    "for r in res.response_gen:\n",
+    "    print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19b7e12c-0729-4181-acce-53a3a95b67b8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "328123 + 2891230"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bca4c0b2-5165-4943-af1f-d3168ee88fcd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

archive/requirements.txt

@@ -0,0 +1,256 @@
+aiohttp==3.9.1
+aiosignal==1.3.1
+alembic==1.13.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+appnope==0.1.3
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asgiref==3.7.2
+asttokens==2.4.1
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.2.0
+Babel==2.14.0
+backoff==2.2.1
+bcrypt==4.1.2
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blinker==1.7.0
+bs4==0.0.2
+build==1.0.3
+cachetools==5.3.2
+certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.22
+click==8.1.7
+coloredlogs==15.0.1
+comm==0.2.0
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.3
+debugpy==1.8.0
+decorator==5.1.1
+defusedxml==0.7.1
+Deprecated==1.2.14
+dill==0.3.7
+dirtyjson==1.0.8
+distro==1.9.0
+entrypoints==0.4
+exceptiongroup==1.2.0
+executing==2.0.1
+Faker==22.0.0
+fastapi==0.109.0
+fastjsonschema==2.19.1
+favicon==0.7.0
+filelock==3.13.1
+flatbuffers==23.5.26
+fonttools==4.47.0
+fqdn==1.5.1
+frozendict==2.4.0
+frozenlist==1.4.1
+fsspec==2023.12.2
+gitdb==4.0.11
+GitPython==3.1.40
+google-auth==2.27.0
+googleapis-common-protos==1.62.0
+greenlet==3.0.3
+grpcio==1.60.0
+h11==0.14.0
+htbuilder==0.6.2
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.26.0
+huggingface-hub==0.20.1
+humanfriendly==10.0
+humanize==4.9.0
+idna==3.6
+importlib-metadata==6.11.0
+importlib-resources==6.1.1
+ipykernel==6.28.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.14
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.20.0
+jsonschema-specifications==2023.12.1
+jupyter==1.0.0
+jupyter-console==6.6.3
+jupyter-events==0.9.0
+jupyter-lsp==2.2.1
+jupyter_client==8.6.0
+jupyter_core==5.6.1
+jupyter_server==2.12.1
+jupyter_server_terminals==0.5.1
+jupyterlab==4.0.10
+jupyterlab-widgets==3.0.9
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.25.2
+kiwisolver==1.4.5
+kubernetes==29.0.0
+langchain==0.0.354
+langchain-community==0.0.8
+langchain-core==0.1.5
+langsmith==0.0.77
+llama-index==0.9.39
+Mako==1.3.0
+Markdown==3.5.1
+markdown-it-py==3.0.0
+markdownlit==0.0.7
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.2
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+merkle-json==1.0.0
+millify==0.1.1
+mistune==3.0.2
+mmh3==4.1.0
+monotonic==1.6
+more-itertools==10.1.0
+mpmath==1.3.0
+multidict==6.0.4
+munch==4.0.0
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.14.0
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+nltk==3.8.1
+notebook==7.0.6
+notebook_shim==0.2.3
+numpy==1.26.2
+oauthlib==3.2.2
+onnxruntime==1.16.3
+openai==1.6.1
+opentelemetry-api==1.22.0
+opentelemetry-exporter-otlp-proto-common==1.22.0
+opentelemetry-exporter-otlp-proto-grpc==1.22.0
+opentelemetry-instrumentation==0.43b0
+opentelemetry-instrumentation-asgi==0.43b0
+opentelemetry-instrumentation-fastapi==0.43b0
+opentelemetry-proto==1.22.0
+opentelemetry-sdk==1.22.0
+opentelemetry-semantic-conventions==0.43b0
+opentelemetry-util-http==0.43b0
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.4
+pandocfilters==1.5.0
+parso==0.8.3
+pexpect==4.9.0
+pillow==10.2.0
+platformdirs==4.1.0
+posthog==3.3.3
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.25.1
+psutil==5.9.7
+ptyprocess==0.7.0
+pulsar-client==3.4.0
+pure-eval==0.2.2
+pyarrow==14.0.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pycparser==2.21
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydeck==0.8.1b0
+Pygments==2.17.2
+pymdown-extensions==10.7
+pyparsing==3.1.1
+pypdf==3.17.4
+PyPika==0.48.9
+pyproject_hooks==1.0.0
+python-dateutil==2.8.2
+python-decouple==3.8
+python-dotenv==1.0.0
+python-json-logger==2.0.7
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==25.1.2
+qtconsole==5.5.1
+QtPy==2.4.1
+referencing==0.32.0
+regex==2023.12.25
+requests==2.31.0
+requests-oauthlib==1.3.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.0
+rpds-py==0.16.2
+rsa==4.9
+safetensors==0.4.1
+scikit-learn==1.4.0
+scipy==1.12.0
+Send2Trash==1.8.2
+sentence-transformers==2.3.0
+sentencepiece==0.1.99
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.24
+st-annotated-text==4.0.1
+stack-data==0.6.3
+starlette==0.35.1
+streamlit==1.29.0
+streamlit-aggrid==0.3.4.post3
+streamlit-camera-input-live==0.2.0
+streamlit-card==1.0.0
+streamlit-embedcode==0.1.2
+streamlit-extras==0.3.6
+streamlit-faker==0.0.3
+streamlit-feedback==0.1.3
+streamlit-image-coordinates==0.1.6
+streamlit-keyup==0.2.2
+streamlit-toggle-switch==1.0.2
+streamlit-vertical-slider==2.5.5
+sympy==1.12
+tenacity==8.2.3
+terminado==0.18.0
+threadpoolctl==3.2.0
+tiktoken==0.5.2
+tinycss2==1.2.1
+tokenizers==0.15.0
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+torch==2.1.2
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.14.0
+transformers==4.36.2
+trulens==0.13.4
+trulens-eval==0.20.0
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+tzlocal==5.2
+uri-template==1.3.0
+urllib3==2.1.0
+uvicorn==0.27.0
+uvloop==0.19.0
+validators==0.22.0
+watchfiles==0.21.0
+wcwidth==0.2.12
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+websockets==12.0
+widgetsnbextension==4.0.9
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.17.0

archive/streamlit_app/streamlit_app_14Feb2024.py

@@ -0,0 +1,288 @@
+import streamlit as st
+from streamlit_feedback import streamlit_feedback
+
+import os
+import pandas as pd
+import base64
+from io import BytesIO
+import nest_asyncio
+
+import chromadb
+from llama_index.legacy import (
+            VectorStoreIndex, 
+            SimpleDirectoryReader,
+            ServiceContext,
+            Document
+)
+
+from llama_index.legacy.vector_stores.chroma import ChromaVectorStore
+from llama_index.legacy.storage.storage_context import StorageContext
+from llama_index.legacy.embeddings import HuggingFaceEmbedding
+from llama_index.legacy.llms import OpenAI
+from llama_index.legacy.memory import ChatMemoryBuffer
+
+from vision_api import get_transcribed_text
+
+nest_asyncio.apply()
+
+# App title
+st.set_page_config(page_title="💬 Open AI Chatbot")
+openai_api = os.getenv("OPENAI_API_KEY")
+
+# "./raw_documents/HI_Knowledge_Base.pdf"
+input_files = ["./raw_documents/HI Chapter Summary Version 1.3.pdf",
+               "./raw_documents/qna.txt"]
+embedding_model = "BAAI/bge-small-en-v1.5"
+persisted_vector_db = "./models/chroma_db"
+fine_tuned_path = "local:models/fine-tuned-embeddings"
+system_content = ("You are a helpful study assistant. "
+                  "You do not respond as 'User' or pretend to be 'User'. "
+                  "You only respond once as 'Assistant'."
+)
+
+data_df = pd.DataFrame(
+    {
+        "Completion": [30, 40, 100, 10],
+    }
+)
+data_df.index = ["Chapter 1", "Chapter 2", "Chapter 3", "Chapter 4"]
+
+# Replicate Credentials
+with st.sidebar:
+    st.title("💬 Open AI Chatbot")
+    st.write("This chatbot is created using the GPT model from Open AI.")
+    if openai_api:
+        pass
+    elif "OPENAI_API_KEY" in st.secrets:
+        st.success("API key already provided!", icon="✅")
+        openai_api = st.secrets["OPENAI_API_KEY"]
+    else:
+        openai_api = st.text_input("Enter OpenAI API token:", type="password")
+        if not (openai_api.startswith("sk-") and len(openai_api)==51):
+            st.warning("Please enter your credentials!", icon="⚠️")
+        else:
+            st.success("Proceed to entering your prompt message!", icon="👉")
+
+    ### for streamlit purpose
+    os.environ["OPENAI_API_KEY"] = openai_api
+
+    st.subheader("Models and parameters")
+    selected_model = st.sidebar.selectbox("Choose an OpenAI model", 
+                                          ["gpt-3.5-turbo-1106", "gpt-4-1106-preview"], 
+                                           key="selected_model")
+    temperature = st.sidebar.slider("temperature", min_value=0.0, max_value=2.0, 
+                                    value=0.0, step=0.01)
+    st.data_editor(
+        data_df,
+        column_config={
+            "Completion": st.column_config.ProgressColumn(
+                            "Completion %",
+                            help="Percentage of content covered",
+                            format="%.1f%%",
+                            min_value=0,
+                            max_value=100,
+            ),
+        },
+        hide_index=False,
+    )
+
+    st.markdown("📖 Reach out to SakiMilo to learn how to create this app!")
+
+if "init" not in st.session_state.keys():
+    st.session_state.init = {"warm_started": "No"}
+    st.session_state.feedback = False
+
+# Store LLM generated responses
+if "messages" not in st.session_state.keys():
+    st.session_state.messages = [{"role": "assistant", 
+                                  "content": "How may I assist you today?",
+                                  "type": "text"}]
+
+if "feedback_key" not in st.session_state:
+    st.session_state.feedback_key = 0
+
+if "release_file" not in st.session_state:
+    st.session_state.release_file = "false"
+
+def clear_chat_history():
+    st.session_state.messages = [{"role": "assistant", 
+                                  "content": "How may I assist you today?",
+                                  "type": "text"}]
+    chat_engine = get_query_engine(input_files=input_files, 
+                                   llm_model=selected_model, 
+                                   temperature=temperature,
+                                   embedding_model=embedding_model,
+                                   fine_tuned_path=fine_tuned_path,
+                                   system_content=system_content,
+                                   persisted_path=persisted_vector_db)
+    chat_engine.reset()
+
+st.sidebar.button("Clear Chat History", on_click=clear_chat_history)
+if st.sidebar.button("I want to submit a feedback!"):
+    st.session_state.feedback = True
+    st.session_state.feedback_key += 1  # overwrite feedback component
+
+@st.cache_resource
+def get_document_object(input_files):
+    documents = SimpleDirectoryReader(input_files=input_files).load_data()
+    document = Document(text="\n\n".join([doc.text for doc in documents]))
+    return document
+
+@st.cache_resource
+def get_llm_object(selected_model, temperature):
+    llm = OpenAI(model=selected_model, temperature=temperature)
+    return llm
+
+@st.cache_resource
+def get_embedding_model(model_name, fine_tuned_path=None):
+    if fine_tuned_path is None:
+        print(f"loading from `{model_name}` from huggingface")
+        embed_model = HuggingFaceEmbedding(model_name=model_name)
+    else:
+        print(f"loading from local `{fine_tuned_path}`")
+        embed_model = fine_tuned_path
+    return embed_model
+
+@st.cache_resource
+def get_query_engine(input_files, llm_model, temperature,
+                     embedding_model, fine_tuned_path,
+                     system_content, persisted_path):
+    
+    llm = get_llm_object(llm_model, temperature)
+    embedded_model = get_embedding_model(
+                        model_name=embedding_model, 
+                        fine_tuned_path=fine_tuned_path
+    )
+    service_context = ServiceContext.from_defaults(
+                        llm=llm, 
+                        embed_model=embedded_model
+    )
+    
+    if os.path.exists(persisted_path):
+        print("loading from vector database - chroma")
+        db = chromadb.PersistentClient(path=persisted_path)
+        chroma_collection = db.get_or_create_collection("quickstart")
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        storage_context = StorageContext.from_defaults(
+                                vector_store=vector_store
+        )
+        index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store, 
+            service_context=service_context, 
+            storage_context=storage_context
+        )
+    else:
+        print("create in-memory vector store")
+        document = get_document_object(input_files)
+        index = VectorStoreIndex.from_documents(
+                    [document], 
+                    service_context=service_context
+        )
+    
+    memory = ChatMemoryBuffer.from_defaults(token_limit=15000)
+    chat_engine = index.as_chat_engine(
+        chat_mode="context",
+        memory=memory,
+        system_prompt=system_content
+    )
+
+    return chat_engine
+
+def generate_llm_response(prompt_input):
+    chat_engine = get_query_engine(input_files=input_files, 
+                                   llm_model=selected_model, 
+                                   temperature=temperature,
+                                   embedding_model=embedding_model,
+                                   fine_tuned_path=fine_tuned_path,
+                                   system_content=system_content,
+                                   persisted_path=persisted_vector_db)
+    
+    # st.session_state.messages
+    response = chat_engine.stream_chat(prompt_input)
+    return response
+
+def handle_feedback(user_response):
+    st.toast("✔️ Feedback received!")
+    st.session_state.feedback = False
+
+def handle_image_upload():
+    st.session_state.release_file = "true"
+
+# Warm start
+if st.session_state.init["warm_started"] == "No":
+    clear_chat_history()
+    st.session_state.init["warm_started"] = "Yes"
+
+# Image upload option
+with st.sidebar:
+    image_file = st.file_uploader("Upload your image here...", 
+                                  type=["png", "jpeg", "jpg"],
+                                  on_change=handle_image_upload)
+
+    if st.session_state.release_file == "true" and image_file:
+        with st.spinner("Uploading..."):
+            b64string = base64.b64encode(image_file.read()).decode('utf-8')
+            message = {
+                    "role": "user", 
+                    "content": b64string,
+                    "type": "image"}
+            st.session_state.messages.append(message)
+
+            transcribed_msg = get_transcribed_text(b64string)
+            message = {
+                    "role": "admin", 
+                    "content": transcribed_msg,
+                    "type": "text"}
+            st.session_state.messages.append(message)
+            st.session_state.release_file = "false"
+
+# Display or clear chat messages
+for message in st.session_state.messages:
+    if message["role"] == "admin":
+        continue
+    with st.chat_message(message["role"]):
+        if message["type"] == "text":
+            st.write(message["content"])
+        elif message["type"] == "image":
+            img_io = BytesIO(base64.b64decode(message["content"].encode("utf-8")))
+            st.image(img_io)
+
+# User-provided prompt
+if prompt := st.chat_input(disabled=not openai_api):
+    client = OpenAI()
+    st.session_state.messages.append({"role": "user", 
+                                      "content": prompt, 
+                                      "type": "text"})
+    with st.chat_message("user"):
+        st.write(prompt)
+
+# Retrieve text prompt from image submission
+if prompt is None and \
+   st.session_state.messages[-1]["role"] == "admin":
+    prompt = st.session_state.messages[-1]["content"]
+
+# Generate a new response if last message is not from assistant
+if st.session_state.messages[-1]["role"] != "assistant":
+    with st.chat_message("assistant"):
+        with st.spinner("Thinking..."):
+            response = generate_llm_response(prompt)
+            placeholder = st.empty()
+            full_response = ""
+            for token in response.response_gen:
+                full_response += token
+                placeholder.markdown(full_response)
+            placeholder.markdown(full_response)
+
+    message = {"role": "assistant", 
+               "content": full_response,
+               "type": "text"}
+    st.session_state.messages.append(message)
+
+# Trigger feedback
+if st.session_state.feedback:
+    result = streamlit_feedback(
+                feedback_type="thumbs",
+                optional_text_label="[Optional] Please provide an explanation",
+                on_submit=handle_feedback,
+                key=f"feedback_{st.session_state.feedback_key}"
+    )

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d37c44e68139700bd5cfddc1f64e610ae6d974b559548175754eac7df1ac8065
-size 1676000
+oid sha256:f6cf8f7e78059a729ff1d88eca662790e198f689fd928b4f862a1aaaf93295e4
+size 25140000

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/header.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e87a1dc8bcae6f2c4bea6d5dd5005454d4dace8637dae29bff3c037ea771411e
+oid sha256:0a95628a4a7b1b1b04e3993394511109b03e9060612be3ebc5a8c488e1cada5c
 size 100

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80795638f4e5e55013e63f7f283e2b3b97b5ad2ac54e10c35cd9564b50c2ba94
+size 868175

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc19b1997119425765295aeab72d76faa6927d4f83985d328c26f20468d6cc76
-size 4000
+oid sha256:dbcafc88fc032c4c88850545971753cc8c8b24837ee14c55cce0839d49055d89
+size 60000

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e08b2de8cb48360a38c01ab6cb218345297bfa7bcf83739723d00a74baa8161e
+size 131332

models/chroma_db/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffe0f3842c7835daddb5c11b8f70bb5dc6352abcb91c11f30c53a49d8c6d540c
-size 23486464
+oid sha256:29a8ddc3f6f6744fe7cce8314b9c7dd60545b66e3bfd25ad7988b47afa93cc94
+size 250535936

notebooks/002_persisted-embedding-model.ipynb

@@ -0,0 +1,476 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8acae3ed-2953-45a3-aba9-0327b6ae3679",
+   "metadata": {},
+   "source": [
+    "### ChromaDB method - create vectorstore based on Chroma"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7de9c591-5a77-4bbe-80f1-4897e15f0b97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.core import ServiceContext\n",
+    "from llama_index.core import Document\n",
+    "\n",
+    "from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()\n",
+    "\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e65dff6-77b6-4be8-8857-5cecf3a035bb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load some documents\n",
+    "documents = SimpleDirectoryReader(input_files=[\n",
+    "                                    \"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "                                    \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "                                    \"../raw_documents/qna.txt\"\n",
+    "                                  ]).load_data()\n",
+    "document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bd86b3f5-1dfc-4257-bd9c-86d34f02398d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# initialize client, setting path to save data\n",
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f568ce7b-bcbf-455c-acf1-6c2cae129fed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create collection\n",
+    "chroma_collection = db.get_or_create_collection(\"quickstart\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ed0b018e-1982-46b2-b1b4-04f5c0ce8672",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# assign chroma as the vector_store to the context\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eb5edab2-30db-4bf7-96b5-4005d3161988",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0946b6ce-96ab-44de-ad75-e424a8429f67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Settings.llm = None\n",
+    "Settings.chunk_size = 1024\n",
+    "Settings.embed_model = \"local:../models/fine-tuned-embeddings\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b8c73a2c-1129-406a-8046-085afcaf9cbb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nodes = Settings.node_parser.get_nodes_from_documents(documents)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adfe688f-95c0-477c-a9de-e9e77541a1d7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dab4c6f3-ef67-4d90-b3d5-e290c5d1b6f4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6a764113-ad7e-4674-aa57-ebbf405902a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context.docstore.add_documents(nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38e7c88d-6c45-4275-8293-d09b4b85a7cf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e492ed4a-23a3-47d6-8b50-51fb48b3aa05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start_time = time.time()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cbd11b89-9b83-4f08-bb30-160f750f2ffb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_index = VectorStoreIndex(nodes, storage_context=storage_context)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "082a0d7e-b025-4db1-be2a-7a0b7bc453b9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_query_engine = vector_index.as_query_engine()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d3bd848d-9985-4a3d-bdc4-ec340cc69ef3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "indexing_cost = time.time() - start_time\n",
+    "indexing_cost = indexing_cost / 60\n",
+    "print(f\"Indexing time: {indexing_cost:.1f} mins\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3290e870-41d7-49c4-9c4f-cb16bd1f469e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = vector_query_engine.query(\"what is the healthcare philosophy in singapore\")\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "131d907a-0677-4ad8-b3f7-6fc9b9c5d0a5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "08fb2be5-3a44-4bb8-a9fc-61d7f03b7a35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7fc01f6-4738-415b-a96b-afd6cf8d789a",
+   "metadata": {},
+   "source": [
+    "### ChromaDB method - load vectorstore based on Chroma"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1a42c35-5f57-423c-8fb7-7d18b3b466b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.core import ServiceContext\n",
+    "from llama_index.core import Document\n",
+    "\n",
+    "from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.core.memory import ChatMemoryBuffer\n",
+    "\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72dd0ece-c72d-428a-89b4-9494d948c845",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d38dc953-b923-4128-86a1-c8c6f69af0ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fine_tuned_path = \"local:../models/fine-tuned-embeddings\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4c83c613-2cfc-4871-9d07-c82f77a3bd5e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(model=\"gpt-3.5-turbo-1106\", temperature=0.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0583e9b0-d977-488c-8331-46dfa749924c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Settings.llm = llm\n",
+    "Settings.embed_model = fine_tuned_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f994f440-f647-48b4-a517-46a79f7561e5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2159a2b6-494b-41b9-ac54-dd342bfb74ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b385644-b46e-4d13-88fa-9f4af39db405",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chroma_collection = db.get_or_create_collection(\"quickstart\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93cb53d1-6b8c-4b2d-a839-53501c0d54b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# assign chroma as the vector_store to the context\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c40d59e1-6d42-41f0-8c9b-70aa026093ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create your index\n",
+    "index = VectorStoreIndex.from_vector_store(\n",
+    "    vector_store=vector_store,\n",
+    "    storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73ba6d06-ba69-4b5e-962a-9cf7d2dc4d94",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a506940-c2b4-4d14-ad93-fd451331c582",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_content = (\"You are a helpful study assistant. \"\n",
+    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
+    "                  \"You only respond once as 'Assistant'.\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3f592848-8536-4b4d-b34a-adc32d043432",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "memory = ChatMemoryBuffer.from_defaults(token_limit=15000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6c7df81a-fd2f-42bf-b09c-46d7750f7252",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chat_engine = index.as_chat_engine(\n",
+    "    chat_mode=\"context\",\n",
+    "    memory=memory,\n",
+    "    system_prompt=system_content\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "434f0caf-8b1f-40c6-b9ec-b039cd1ca612",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = chat_engine.chat(\"what is the healthcare philosophy in singapore\")\n",
+    "print(res.response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e62303c-3a00-448f-ad93-15cb6cee1f24",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dad72f9f-7f86-407d-93be-f5724cb30d5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_engine = index.as_query_engine(\n",
+    "                memory=memory,\n",
+    "                system_prompt=system_content,\n",
+    "                similarity_top_k=3,\n",
+    "                streaming=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab778a5d-d438-4f39-88f5-c67a1f1d575e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7bb7c21a-7461-40c1-87a7-4a1f92f70153",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = hi_engine.query(\"What is llama2?\")\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "874a39ce-e682-42fa-8085-646bacea6cdb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "301e8270-783d-4942-a05f-9683ca96fbda",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/{qna_prompting_with_function_calling.ipynb → 004_qna_prompting_with_function_calling.ipynb}

@@ -339,6 +339,32 @@
     "print(json_response)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "049a28bf-abe5-4247-970f-615d1877a2c0",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de49c61a-0b3e-4623-abcb-a7625ac4d0db",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"I am interested in 2nd chapter, can you test my understanding of this chapter?\"\n",
+    "response = client.chat.completions.create(\n",
+    "    model = 'gpt-3.5-turbo',\n",
+    "    messages = [{'role': 'user', 'content': prompt}],\n",
+    "    functions = qna_custom_functions,\n",
+    "    function_call = 'auto'\n",
+    ")\n",
+    "json_response = json.loads(response.choices[0].message.function_call.arguments)\n",
+    "print(json_response)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

notebooks/005_qna_prompting_with_pydantic_embeddings.ipynb

@@ -0,0 +1,676 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4cf3ca7c-2c43-495b-a1ee-24c770f0ad1e",
+   "metadata": {},
+   "source": [
+    "### Simple OpenAI agent with tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a9d74b11-4049-4e3c-839e-7d13d7c0dadc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import sqlite3\n",
+    "from typing import Sequence, List\n",
+    "from pydantic import BaseModel, Field\n",
+    "\n",
+    "from llama_index.core.llms import ChatMessage\n",
+    "from llama_index.core.tools import BaseTool, FunctionTool\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.agent.openai import OpenAIAgent\n",
+    "\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "33da3ba3-7d5d-4528-8e31-cabf85a88886",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b27b2c3b-1876-4a20-8fb9-cddd6df51ef3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db_path = \"../database/mock_qna.db\"\n",
+    "con = sqlite3.connect(db_path)\n",
+    "cur = con.cursor()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b69adba2-9f98-460b-b0a3-e759d6ac1b88",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39ca840b-0ec7-4279-9654-e4ddfda6137f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define sample Tool\n",
+    "def multiply(a: int, b: int) -> int:\n",
+    "    \"\"\"Multiply two integers and returns the result integer\"\"\"\n",
+    "    return a * b\n",
+    "\n",
+    "multiply_tool = FunctionTool.from_defaults(fn=multiply)\n",
+    "\n",
+    "def add(a: int, b: int) -> int:\n",
+    "    \"\"\"Add two integers and returns the result integer\"\"\"\n",
+    "    return a + b\n",
+    "\n",
+    "add_tool = FunctionTool.from_defaults(fn=add)\n",
+    "\n",
+    "class Song(BaseModel):\n",
+    "    \"\"\"A song with name and artist\"\"\"\n",
+    "\n",
+    "    name: str\n",
+    "    artist: str\n",
+    "\n",
+    "song_fn = FunctionTool.from_defaults(fn=Song)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14b64612-1320-48b2-b5ff-91cde659cbf8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class QnA_Model(BaseModel):\n",
+    "    chapter_n: str = Field(..., \n",
+    "                           pattern=r'^Chapter_\\d*$',\n",
+    "                           description=(\n",
+    "                                \"which chapter to extract, the format of this function argumet\"\n",
+    "                                \"is with `Chapter_` as prefix concatenated with chapter number\"\n",
+    "                                \"in integer. For example, `Chapter_2`, `Chapter_10`.\")\n",
+    "                          )\n",
+    "\n",
+    "description = \"\"\"\n",
+    "      Use this tool to extract the chapter information from the body of the input text, \n",
+    "      the format looks as follow:\n",
+    "        The output should be in the format with `Chapter_` as prefix.\n",
+    "        Example 1: `Chapter_1` for first chapter\n",
+    "        Example 2: For chapter 12 of the textbook, you should return `Chapter_12`\n",
+    "        Example 3: `Chapter_5` for fifth chapter\n",
+    "        Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.\n",
+    "\"\"\"\n",
+    "\n",
+    "def get_qna_question(chapter_n: str):\n",
+    "    \"\"\"\n",
+    "      Use this tool to extract the chapter information from the body of the input text, \n",
+    "      the format looks as follow:\n",
+    "        The output should be in the format with `Chapter_` as prefix.\n",
+    "        Example 1: `Chapter_1` for first chapter\n",
+    "        Example 2: For chapter 12 of the textbook, you should return `Chapter_12`\n",
+    "        Example 3: `Chapter_5` for fifth chapter\n",
+    "        Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.\n",
+    "    \"\"\"\n",
+    "    sql_string = f\"\"\"SELECT id, question, option_1, option_2, option_3, option_4, correct_answer\n",
+    "                     FROM qna_tbl\n",
+    "                     WHERE chapter='{chapter_n}'\n",
+    "                  \"\"\"\n",
+    "    res = cur.execute(sql_string)\n",
+    "    result = res.fetchone()\n",
+    "\n",
+    "    id       = result[0]\n",
+    "    question = result[1]\n",
+    "    option_1 = result[2]\n",
+    "    option_2 = result[3]\n",
+    "    option_3 = result[4]\n",
+    "    option_4 = result[5]\n",
+    "    c_answer = result[6]\n",
+    "\n",
+    "    qna_str  = \"Question: \\n\" + \\\n",
+    "               \"========= \\n\" + \\\n",
+    "                question.replace(\"\\\\n\", \"\\n\") + \"\\n\" + \\\n",
+    "               \"A) \" + option_1 + \"\\n\" + \\\n",
+    "               \"B) \" + option_2 + \"\\n\" + \\\n",
+    "               \"C) \" + option_3 + \"\\n\" + \\\n",
+    "               \"D) \" + option_4\n",
+    "    \n",
+    "    # return id, qna_str, c_answer\n",
+    "    return qna_str\n",
+    "\n",
+    "get_qna_question_tool = FunctionTool.from_defaults(\n",
+    "                            fn=get_qna_question,\n",
+    "                            name=\"Extract_Question\",\n",
+    "                            description=description,\n",
+    "                            fn_schema=QnA_Model\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4df3e9f2-4a32-4449-b203-929dff9e7963",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbeeea36-0bb0-4edb-9b8c-adb7c64c4cd3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# initialize openai agent\n",
+    "llm = OpenAI(model=\"gpt-3.5-turbo-0613\")\n",
+    "agent = OpenAIAgent.from_tools([multiply_tool, \n",
+    "                                add_tool, \n",
+    "                                song_fn, \n",
+    "                                get_qna_question_tool], llm=llm, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0edafe7d-a835-4882-bd7d-1717a4cad462",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65158ede-b99f-477d-9d17-3be40e57a629",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"3 x 2 equals?\", tool_choice=\"auto\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d74b0e5c-47a2-4de4-acd2-d39a94941f2d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9787ed4-46a2-46aa-80e6-b317d9280b9e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"3 plus 2 equals?\", tool_choice=\"auto\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dd3358c6-e0e4-4354-8a4e-97d70254f648",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16e1db2c-dca3-4dc6-9cc5-c10644d5927c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"give me the lyrics of taylor swift's `you belong with me`\", tool_choice=\"auto\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "97a021f4-4701-4914-9ab8-0683b396f096",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1f0b352d-8510-4b2a-a495-9f2e1fbfcddb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# res_stream = agent.stream_chat(message=\"3 x 2 equals?\")\n",
+    "# for r in res_stream.response_gen:\n",
+    "#     print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea0a6cd4-f204-4997-bdfb-cb9b5a9e1266",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "96c978e6-62e2-46e6-ae63-76841487f618",
+   "metadata": {},
+   "source": [
+    "### OpenAI agent with embeddings, and function calling"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f0f5f02-c8e9-43a9-853d-12bb3c19dbe8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import chromadb\n",
+    "\n",
+    "from llama_index.core import (\n",
+    "    SimpleDirectoryReader,\n",
+    "    VectorStoreIndex,\n",
+    "    ServiceContext,\n",
+    "    StorageContext,\n",
+    "    load_index_from_storage,\n",
+    ")\n",
+    "from llama_index.core.memory import ChatMemoryBuffer\n",
+    "from llama_index.vector_stores.chroma.base import ChromaVectorStore\n",
+    "from llama_index.core.tools import QueryEngineTool, ToolMetadata\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "from llama_index.agent.openai import OpenAIAgent\n",
+    "from llama_index.core.tools import BaseTool, FunctionTool\n",
+    "from llama_index.core import Settings\n",
+    "\n",
+    "from pydantic import BaseModel\n",
+    "import nest_asyncio\n",
+    "\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54855aa4-dcad-404e-843f-c96d61046df3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4edb89f6-bb2f-46ff-8807-dfb03115fcd5",
+   "metadata": {},
+   "source": [
+    "#### Parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "61ad7369-8fd4-434f-b687-0c649940bda1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected_model = \"gpt-3.5-turbo-1106\"\n",
+    "temperature = 0.0\n",
+    "\n",
+    "persisted_vector_db = \"../models/chroma_db\"\n",
+    "input_files = [\"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "               \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "               \"../raw_documents/qna.txt\"]\n",
+    "fine_tuned_path = \"local:../models/fine-tuned-embeddings\"\n",
+    "system_content = (\"You are a helpful study assistant. \"\n",
+    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
+    "                  \"You only respond once as 'Assistant'.\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a18117f5-48a7-4e81-9b46-541f382caf9e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3210c837-9b40-4cd9-bb00-ead559deff6f",
+   "metadata": {},
+   "source": [
+    "#### Load vector store"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9dfba0c-f27d-49d1-86c5-a1d95c11b844",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if os.path.exists(persisted_vector_db):\n",
+    "    db = chromadb.PersistentClient(path=persisted_vector_db)\n",
+    "    chroma_collection = db.get_or_create_collection(\"quickstart\")\n",
+    "    \n",
+    "    # assign chroma as the vector_store to the context\n",
+    "    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "    storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "else:\n",
+    "    documents = SimpleDirectoryReader(input_files=input_files).load_data()\n",
+    "    document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))\n",
+    "    \n",
+    "    # initialize client, setting path to save data\n",
+    "    db = chromadb.PersistentClient(path=persisted_vector_db)\n",
+    "    \n",
+    "    # create collection\n",
+    "    chroma_collection = db.get_or_create_collection(\"quickstart\")\n",
+    "\n",
+    "    # assign chroma as the vector_store to the context\n",
+    "    vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "    storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "693c9808-efbe-47a6-a49c-7143c63d13e5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c6bfae0b-7c97-4c2b-9996-f5e3ecf7a992",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define sample Tool\n",
+    "def multiply(a: int, b: int) -> int:\n",
+    "    \"\"\"Multiply two integers and returns the result integer\"\"\"\n",
+    "    return a * b\n",
+    "\n",
+    "def add(a: int, b: int) -> int:\n",
+    "    \"\"\"Add two integers and returns the result integer\"\"\"\n",
+    "    return a + b\n",
+    "\n",
+    "class Song(BaseModel):\n",
+    "    \"\"\"A song with name and artist\"\"\"\n",
+    "\n",
+    "    name: str\n",
+    "    artist: str\n",
+    "\n",
+    "add_tool = FunctionTool.from_defaults(fn=add)\n",
+    "multiply_tool = FunctionTool.from_defaults(fn=multiply)\n",
+    "song_fn = FunctionTool.from_defaults(fn=Song)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16a80b2e-8e5f-462a-8616-042afe18be3a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(model=selected_model, temperature=temperature)\n",
+    "\n",
+    "Settings.llm = llm\n",
+    "Settings.embed_model = fine_tuned_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "95d3a420-1ee4-45bd-a18b-b398d9531db4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = VectorStoreIndex.from_vector_store(\n",
+    "    vector_store=vector_store,\n",
+    "    storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eac6d76d-059b-40e3-b67f-c736f1ce6baa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "memory = ChatMemoryBuffer.from_defaults(token_limit=15000)\n",
+    "\n",
+    "hi_engine = index.as_query_engine(\n",
+    "                memory=memory,\n",
+    "                system_prompt=system_content,\n",
+    "                similarity_top_k=3,\n",
+    "                streaming=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18e38a8f-9b51-4675-a1d5-8aaa6c21694c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = hi_engine.query(\"what is the healthcare philosophy in singapore\")\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70dae6f7-682e-42d6-be59-3b807c10482c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1eb5df65-c926-4b22-8071-449d645b339f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hi_query_tool = QueryEngineTool.from_defaults(\n",
+    "                    query_engine=hi_engine,\n",
+    "                    name=\"vector_tool\",\n",
+    "                    description=(\n",
+    "                        \"Provides information about Health Insurance landscape in Singapore. \"\n",
+    "                        \"Use a detailed plain text question as input to the tool.\"\n",
+    "                    )\n",
+    ")\n",
+    "\n",
+    "# hi_query_tool = QueryEngineTool(\n",
+    "#                     query_engine=hi_engine,\n",
+    "#                     metadata=ToolMetadata(\n",
+    "#                         name=\"health_insurance_mentor\",\n",
+    "#                         description=(\n",
+    "#                             \"Provides information about Health Insurance landscape in Singapore. \"\n",
+    "#                             \"Use a detailed plain text question as input to the tool.\"\n",
+    "#                         )\n",
+    "#                     )\n",
+    "#                 )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5822b1d-32ef-4b68-8629-a727ff51cd0a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2a1235da-a379-4055-8bcf-4b21c91c9fb0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent = OpenAIAgent.from_tools([multiply_tool, \n",
+    "                                add_tool, \n",
+    "                                hi_query_tool, \n",
+    "                                get_qna_question_tool], llm=llm, verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05b65cbd-d15c-4909-b383-50b13f64e535",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63332a44-9441-4f49-85a2-934e2c55a362",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(\"what is the healthcare philosophy in singapore\", tool_choice=\"auto\")\n",
+    "res.response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de387041-706c-4be8-ab31-fe8bd8b16bc1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eb8a8676-c070-4652-8c00-436be3135c12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.chat(message=\"I am interested in 4th chapter, can you test my understanding of this chapter?\",\n",
+    "                 tool_choice=\"auto\")\n",
+    "res.response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81709cbf-9a5e-482f-ae6a-ba361b8219dc",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adf26268-e40a-4ebd-a737-6b203ddc4444",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.stream_chat(\"what is the healthcare philosophy in singapore\", tool_choice=\"auto\")\n",
+    "for r in res.response_gen:\n",
+    "    print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c4f2df51-553f-493d-874c-662ecb499e36",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4bf66e59-a394-4e6b-b7d5-af6b1612c97b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.stream_chat(message=\"I am interested in 4th chapter, can you test my understanding of this chapter?\",\n",
+    "                        tool_choice=\"auto\")\n",
+    "for r in res.response_gen:\n",
+    "    print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "540c0f71-048a-4a64-9818-e2b1cffc0db7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fbeabf28-30f9-4d7f-a4b9-21cd08a9b128",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = agent.stream_chat(\"what is the result of 328123 + 2891230\", tool_choice=\"auto\")\n",
+    "for r in res.response_gen:\n",
+    "    print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19b7e12c-0729-4181-acce-53a3a95b67b8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "328123 + 2891230"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bca4c0b2-5165-4943-af1f-d3168ee88fcd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/qna_prompting_with_pydantic.ipynb

@@ -1,114 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6f0f5f02-c8e9-43a9-853d-12bb3c19dbe8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pydantic import BaseModel"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "94244a1e-e55a-4954-885e-4558797c6fe3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from llama_index.llms import OpenAI"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "641f36c7-0aa3-4146-9840-bfb0d4d78b4d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from llama_index.core.tools import BaseTool, FunctionTool"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cb20cd13-20fd-4303-acde-b7abe0b48e39",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ab4d1a52-84be-492f-8275-3da20d854cb6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class Song(BaseModel):\n",
-    "    \"\"\"A song with name and artist\"\"\"\n",
-    "\n",
-    "    name: str\n",
-    "    artist: str"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a5822b1d-32ef-4b68-8629-a727ff51cd0a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "63332a44-9441-4f49-85a2-934e2c55a362",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "song_fn = FunctionTool.from_defaults(fn=Song)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ef0d7d67-9855-47ea-8569-7bfb20b03a07",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "response = OpenAI().complete(\"Generate a song\", tools=[song_fn])\n",
-    "tool_calls = response.additional_kwargs[\"tool_calls\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bca4c0b2-5165-4943-af1f-d3168ee88fcd",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.18"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

qna_prompting.py

@@ -0,0 +1,73 @@
+import sqlite3
+from pydantic import BaseModel, Field
+from llama_index.core.tools import FunctionTool
+
+
+db_path = "./database/mock_qna.db"
+description = """
+      Use this tool to extract the chapter information from the body of the input text, 
+      when user wants to learn more about a particular chapter and requested to be asked 
+      with a question to test his/her understanding.
+      The format of the function argument looks as follow:
+        It should be in the format with `Chapter_` as prefix.
+        Example 1: `Chapter_1` for first chapter
+        Example 2: For chapter 12 of the textbook, you should return `Chapter_12`
+        Example 3: `Chapter_5` for fifth chapter
+        Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.
+"""
+
+class QnA_Model(BaseModel):
+    chapter_n: str = Field(..., 
+                           pattern=r'^Chapter_\d*$',
+                           description=(
+                                "which chapter to extract, the format of this function argumet"
+                                "is with `Chapter_` as prefix concatenated with chapter number"
+                                "in integer. For example, `Chapter_2`, `Chapter_10`.")
+                          )
+
+def get_qna_question(chapter_n: str) -> str:
+    """
+      Use this tool to extract the chapter information from the body of the input text, 
+      the format looks as follow:
+        The output should be in the format with `Chapter_` as prefix.
+        Example 1: `Chapter_1` for first chapter
+        Example 2: For chapter 12 of the textbook, you should return `Chapter_12`
+        Example 3: `Chapter_5` for fifth chapter
+        Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.
+    """
+    con = sqlite3.connect(db_path)
+    cur = con.cursor()
+
+    sql_string = f"""SELECT id, question, option_1, option_2, option_3, option_4, correct_answer
+                     FROM qna_tbl
+                     WHERE chapter='{chapter_n}'
+                  """
+    res = cur.execute(sql_string)
+    result = res.fetchone()
+
+    id       = result[0]
+    question = result[1]
+    option_1 = result[2]
+    option_2 = result[3]
+    option_3 = result[4]
+    option_4 = result[5]
+    c_answer = result[6]
+
+    qna_str  = "Question: \n" + \
+               "========= \n" + \
+                question.replace("\\n", "\n") + "\n" + \
+               "A) " + option_1 + "\n" + \
+               "B) " + option_2 + "\n" + \
+               "C) " + option_3 + "\n" + \
+               "D) " + option_4
+    
+    con.close()
+    
+    return qna_str
+
+get_qna_question_tool = FunctionTool.from_defaults(
+                            fn=get_qna_question,
+                            name="Extract_Question",
+                            description=description,
+                            fn_schema=QnA_Model
+)

requirements.txt

@@ -101,7 +101,19 @@ langchain==0.0.354
 langchain-community==0.0.8
 langchain-core==0.1.5
 langsmith==0.0.77
-llama-index==0.9.39
+llama-index==0.10.1
+llama-index-agent-openai==0.1.1
+llama-index-core==0.10.1
+llama-index-embeddings-huggingface==0.1.1
+llama-index-embeddings-openai==0.1.1
+llama-index-legacy==0.9.48
+llama-index-llms-openai==0.1.1
+llama-index-multi-modal-llms-openai==0.1.1
+llama-index-program-openai==0.1.1
+llama-index-question-gen-openai==0.1.1
+llama-index-readers-file==0.1.2
+llama-index-vector-stores-chroma==0.1.1
+lxml==5.1.0
 Mako==1.3.0
 Markdown==3.5.1
 markdown-it-py==3.0.0
@@ -131,7 +143,7 @@ notebook==7.0.6
 notebook_shim==0.2.3
 numpy==1.26.2
 oauthlib==3.2.2
-onnxruntime==1.16.3
+onnxruntime==1.17.0
 openai==1.6.1
 opentelemetry-api==1.22.0
 opentelemetry-exporter-otlp-proto-common==1.22.0
@@ -168,8 +180,10 @@ pydantic_core==2.14.6
 pydeck==0.8.1b0
 Pygments==2.17.2
 pymdown-extensions==10.7
+PyMuPDF==1.23.22
+PyMuPDFb==1.23.22
 pyparsing==3.1.1
-pypdf==3.17.4
+pypdf==4.0.1
 PyPika==0.48.9
 pyproject_hooks==1.0.0
 python-dateutil==2.8.2
@@ -222,7 +236,7 @@ terminado==0.18.0
 threadpoolctl==3.2.0
 tiktoken==0.5.2
 tinycss2==1.2.1
-tokenizers==0.15.0
+tokenizers==0.15.2
 toml==0.10.2
 tomli==2.0.1
 toolz==0.12.0
@@ -230,7 +244,7 @@ torch==2.1.2
 tornado==6.4
 tqdm==4.66.1
 traitlets==5.14.0
-transformers==4.36.2
+transformers==4.37.2
 trulens==0.13.4
 trulens-eval==0.20.0
 typer==0.9.0

streamlit_app.py

@@ -5,21 +5,26 @@ import os
 import pandas as pd
 import base64
 from io import BytesIO
-import nest_asyncio
 
 import chromadb
-from llama_index import (VectorStoreIndex, 
-                         SimpleDirectoryReader,
-                         ServiceContext,
-                         Document)
-from llama_index.vector_stores import ChromaVectorStore
-from llama_index.storage.storage_context import StorageContext
-from llama_index.embeddings import HuggingFaceEmbedding
-from llama_index.llms import OpenAI
-from llama_index.memory import ChatMemoryBuffer
+from llama_index.core import (
+            VectorStoreIndex, 
+            SimpleDirectoryReader,
+            StorageContext,
+            Document
+)
+from llama_index.vector_stores.chroma.base import ChromaVectorStore
+from llama_index.embeddings.huggingface.base import HuggingFaceEmbedding
+from llama_index.llms.openai import OpenAI
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.tools import QueryEngineTool
+from llama_index.agent.openai import OpenAIAgent
+from llama_index.core import Settings
 
 from vision_api import get_transcribed_text
+from qna_prompting import get_qna_question_tool
 
+import nest_asyncio
 nest_asyncio.apply()
 
 # App title
@@ -27,6 +32,7 @@ st.set_page_config(page_title="💬 Open AI Chatbot")
 openai_api = os.getenv("OPENAI_API_KEY")
 
 # "./raw_documents/HI_Knowledge_Base.pdf"
+image_prompt = False
 input_files = ["./raw_documents/HI Chapter Summary Version 1.3.pdf",
                "./raw_documents/qna.txt"]
 embedding_model = "BAAI/bge-small-en-v1.5"
@@ -111,7 +117,7 @@ def clear_chat_history():
                                    embedding_model=embedding_model,
                                    fine_tuned_path=fine_tuned_path,
                                    system_content=system_content,
-                                   persisted_path=persisted_vector_db)
+                                   persisted_vector_db=persisted_vector_db)
     chat_engine.reset()
 
 st.sidebar.button("Clear Chat History", on_click=clear_chat_history)
@@ -143,59 +149,81 @@ def get_embedding_model(model_name, fine_tuned_path=None):
 @st.cache_resource
 def get_query_engine(input_files, llm_model, temperature,
                      embedding_model, fine_tuned_path,
-                     system_content, persisted_path):
+                     system_content, persisted_vector_db):
     
     llm = get_llm_object(llm_model, temperature)
     embedded_model = get_embedding_model(
                         model_name=embedding_model, 
                         fine_tuned_path=fine_tuned_path
     )
-    service_context = ServiceContext.from_defaults(
-                        llm=llm, 
-                        embed_model=embedded_model
-    )
-    
-    if os.path.exists(persisted_path):
+    Settings.llm = llm
+    Settings.chunk_size = 1024
+    Settings.embed_model = embedded_model
+
+    if os.path.exists(persisted_vector_db):
         print("loading from vector database - chroma")
-        db = chromadb.PersistentClient(path=persisted_path)
+        db = chromadb.PersistentClient(path=persisted_vector_db)
         chroma_collection = db.get_or_create_collection("quickstart")
         vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
-        storage_context = StorageContext.from_defaults(
-                                vector_store=vector_store
-        )
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
         index = VectorStoreIndex.from_vector_store(
-            vector_store=vector_store, 
-            service_context=service_context, 
+            vector_store=vector_store,
             storage_context=storage_context
         )
     else:
-        print("create in-memory vector store")
-        document = get_document_object(input_files)
-        index = VectorStoreIndex.from_documents(
-                    [document], 
-                    service_context=service_context
-        )
+        print("create new chroma vector database..")
+        documents = SimpleDirectoryReader(input_files=input_files).load_data()
+        
+        db = chromadb.PersistentClient(path=persisted_vector_db)
+        chroma_collection = db.get_or_create_collection("quickstart")
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        
+        nodes = Settings.node_parser.get_nodes_from_documents(documents)
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        storage_context.docstore.add_documents(nodes)
+
+        index = VectorStoreIndex(nodes, storage_context=storage_context)
     
     memory = ChatMemoryBuffer.from_defaults(token_limit=15000)
-    chat_engine = index.as_chat_engine(
-        chat_mode="context",
-        memory=memory,
-        system_prompt=system_content
+    hi_content_engine = index.as_query_engine(
+                            memory=memory,
+                            system_prompt=system_content,
+                            similarity_top_k=3,
+                            streaming=True
     )
 
-    return chat_engine
-
-def generate_llm_response(prompt_input):
-    chat_engine = get_query_engine(input_files=input_files, 
+    hi_query_tool = QueryEngineTool.from_defaults(
+                        query_engine=hi_content_engine,
+                        name="vector_tool",
+                        description=(
+                            "Provides information about Health Insurance landscape in Singapore. "
+                            "Use a detailed plain text question as input to the tool."
+                        )
+    )
+    agent = OpenAIAgent.from_tools(tools=[
+                                        hi_query_tool, 
+                                        get_qna_question_tool
+                                    ],
+                                   llm=llm, 
+                                   verbose=True)
+    print("loaded AI agent, let's begin the chat!")
+    print("="*50)
+    print("")
+
+    return agent
+
+def generate_llm_response(prompt_input, tool_choice="auto"):
+    chat_agent = get_query_engine(input_files=input_files, 
                                    llm_model=selected_model, 
                                    temperature=temperature,
                                    embedding_model=embedding_model,
                                    fine_tuned_path=fine_tuned_path,
                                    system_content=system_content,
-                                   persisted_path=persisted_vector_db)
+                                   persisted_vector_db=persisted_vector_db)
     
     # st.session_state.messages
-    response = chat_engine.stream_chat(prompt_input)
+    response = chat_agent.stream_chat(prompt_input, tool_choice=tool_choice)
     return response
 
 def handle_feedback(user_response):
@@ -256,13 +284,18 @@ if prompt := st.chat_input(disabled=not openai_api):
 # Retrieve text prompt from image submission
 if prompt is None and \
    st.session_state.messages[-1]["role"] == "admin":
+    image_prompt = True
     prompt = st.session_state.messages[-1]["content"]
 
 # Generate a new response if last message is not from assistant
 if st.session_state.messages[-1]["role"] != "assistant":
     with st.chat_message("assistant"):
         with st.spinner("Thinking..."):
-            response = generate_llm_response(prompt)
+            if image_prompt:
+                response = generate_llm_response(prompt, tool_choice="vector_tool")
+                image_prompt = False
+            else:
+                response = generate_llm_response(prompt, tool_choice="auto")
             placeholder = st.empty()
             full_response = ""
             for token in response.response_gen: