resolved pdf reader issue

config/model_config_advanced.yml

@@ -11,7 +11,7 @@ embeddings:
   fine_tuned_embedding_model: 'local:models/fine-tuned-embeddings-advanced'
 
 vector_store:
-  persisted_path: './models/chroma_db_advanced'
+  persisted_path: './models/chroma_db_advanced_corrected'
 
 questionaire_data:
   db_path: './database/mock_qna.sqlite'

database/mock_qna.sqlite

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23973f07315b6b1f3aaa1d3b90263a9e60518724f8a077cd6fdb44d809db6da4
+oid sha256:d7f3f8c146d46df19f3dd8a4846ccbf63f88e6dd914b67a0c5c689eba21a558d
 size 40960

models/chroma_db_advanced_corrected/7ffa7a70-bc3b-4a97-858f-3c8223492b3e/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c7deba9397301d1ea3f7f5edcb06162bc4797984100c456b128303c58b95c79
+size 31844000

models/chroma_db_advanced_corrected/7ffa7a70-bc3b-4a97-858f-3c8223492b3e/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fdac383bac7a3236c814029cee7525e8018d396f9cd0d15b97a22a3af9090d8
+size 100

models/chroma_db_advanced_corrected/7ffa7a70-bc3b-4a97-858f-3c8223492b3e/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:175eec7edc141af44f996bba0295ddb71d3fd54c39b1352539ede6753f00e834
+size 1100226

models/chroma_db_advanced_corrected/7ffa7a70-bc3b-4a97-858f-3c8223492b3e/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68f24a06f63a85c1b082283b0be703475bf9023e9f4f7e8b3bd4bca276af3b8f
+size 76000

models/chroma_db_advanced_corrected/7ffa7a70-bc3b-4a97-858f-3c8223492b3e/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:224e61d8e99cd63b57c6e9204f3136aba4a43f9bb15d7dcd5eb181c9378829f8
+size 167188

models/chroma_db_advanced_corrected/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf347334a74a91a9d24d298ad1b12fa043579fe0b915cf7bc558f3d2acafca5c
+size 299061248

notebooks/002_persisted-embedding-model-advanced.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "7de9c591-5a77-4bbe-80f1-4897e15f0b97",
    "metadata": {},
    "outputs": [],
@@ -33,24 +33,16 @@
     "import nest_asyncio\n",
     "nest_asyncio.apply()\n",
     "\n",
-    "import time"
+    "import time\n",
+    "import PyPDF2"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "978152ce-4d87-44b5-b521-dbaff60b32b0",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "199it [00:00, 8821.71it/s]\n",
-      "200it [00:00, 12584.17it/s]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "split_content(filepath=\"../raw_documents/answers.txt\", \n",
     "              separator=\"\\n\\n\", \n",
@@ -63,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "d925371b-8777-4f5b-a7f2-ec3f228ef266",
    "metadata": {},
    "outputs": [],
@@ -84,41 +76,64 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "e876a26b-822d-44d6-a3dd-ccdcc04933cf",
+   "id": "a83b4fd8-5075-4c52-820c-a3ac7ee7f0c8",
    "metadata": {},
    "outputs": [],
    "source": []
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "3e65dff6-77b6-4be8-8857-5cecf3a035bb",
    "metadata": {},
    "outputs": [],
    "source": [
     "# load some documents\n",
-    "documents = SimpleDirectoryReader(input_files=[\n",
-    "                                    \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
-    "                                    \"../raw_documents/conversation_examples.txt\",\n",
-    "                                    \"../raw_documents/HI_Knowledge_Base.pdf\",\n",
-    "                                  ] + answers_temp_files + qna_temp_files ).load_data()\n",
-    "document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))"
+    "if False:\n",
+    "    documents = SimpleDirectoryReader(input_files=[\n",
+    "                                        \"../raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
+    "                                        \"../raw_documents/conversation_examples.txt\",\n",
+    "                                        \"../raw_documents/HI_Knowledge_Base.pdf\",\n",
+    "                                      ] + answers_temp_files + qna_temp_files ).load_data()\n",
+    "else:\n",
+    "    reader_summary = PyPDF2.PdfReader(\"../raw_documents/HI Chapter Summary Version 1.3.pdf\")\n",
+    "    documents_summary = [ p.extract_text() for p in reader_summary.pages ]\n",
+    "\n",
+    "    reader_base = PyPDF2.PdfReader(\"../raw_documents/HI_Knowledge_Base.pdf\")\n",
+    "    documents_base = [ p.extract_text() for p in reader_base.pages ]\n",
+    "    \n",
+    "    documents_txt = SimpleDirectoryReader(input_files=[\n",
+    "                                        \"../raw_documents/conversation_examples.txt\",\n",
+    "                                        \"../raw_documents/qna.txt\",\n",
+    "                                        \"../raw_documents/answers.txt\"\n",
+    "                                      ] ).load_data()\n",
+    "    documents_txt = [doc.text for doc in documents_txt]\n",
+    "\n",
+    "document = Document(text=\"\\n\\n\".join(documents_summary + documents_base + documents_txt))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
+   "id": "e485f801-1829-4b50-b6b2-52803203853b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "id": "bd86b3f5-1dfc-4257-bd9c-86d34f02398d",
    "metadata": {},
    "outputs": [],
    "source": [
     "# initialize client, setting path to save data\n",
-    "db = chromadb.PersistentClient(path=\"../models/chroma_db_advanced\")"
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db_advanced_corrected\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "id": "f568ce7b-bcbf-455c-acf1-6c2cae129fed",
    "metadata": {},
    "outputs": [],
@@ -129,7 +144,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "id": "ed0b018e-1982-46b2-b1b4-04f5c0ce8672",
    "metadata": {},
    "outputs": [],
@@ -148,18 +163,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "0946b6ce-96ab-44de-ad75-e424a8429f67",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "LLM is explicitly disabled. Using MockLLM.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "Settings.llm = None\n",
     "Settings.chunk_size = 1024\n",
@@ -169,31 +176,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "b8c73a2c-1129-406a-8046-085afcaf9cbb",
    "metadata": {},
    "outputs": [],
    "source": [
-    "nodes = Settings.node_parser.get_nodes_from_documents(documents)"
+    "nodes = Settings.node_parser.get_nodes_from_documents([document])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "id": "75f1c76f-d3e5-4b69-818c-98865adb1457",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "6814"
-      ]
-     },
-     "execution_count": 13,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "len(nodes)"
    ]
@@ -208,7 +204,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "dab4c6f3-ef67-4d90-b3d5-e290c5d1b6f4",
    "metadata": {},
    "outputs": [],
@@ -218,7 +214,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "6a764113-ad7e-4674-aa57-ebbf405902a8",
    "metadata": {},
    "outputs": [],
@@ -236,7 +232,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "id": "e492ed4a-23a3-47d6-8b50-51fb48b3aa05",
    "metadata": {},
    "outputs": [],
@@ -246,7 +242,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "id": "cbd11b89-9b83-4f08-bb30-160f750f2ffb",
    "metadata": {},
    "outputs": [],
@@ -256,18 +252,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
    "id": "d3bd848d-9985-4a3d-bdc4-ec340cc69ef3",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Indexing time: 2.3 mins\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "indexing_cost = time.time() - start_time\n",
     "indexing_cost = indexing_cost / 60\n",
@@ -276,7 +264,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "id": "f16cca33-71fb-437d-a033-671b9fd44054",
    "metadata": {},
    "outputs": [],
@@ -286,28 +274,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": null,
    "id": "3290e870-41d7-49c4-9c4f-cb16bd1f469e",
    "metadata": {
     "scrolled": true
    },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Response(response='Context information is below.\\n---------------------\\nfile_path: ../raw_documents/answers_temp/answers_050.txt\\n\\nQuestion: The fundamental principle of Singapore healthcare financing is ____________.\\nAnswer: The answer is \"Individual Savings\".\\n\\nfile_path: ../raw_documents/qna_temp/qna_050.txt\\n\\nC1/5\\nQuestion: The fundamental principle of Singapore healthcare financing is ____________.\\nA. The 3’s M. That is Medisave, Medishield, Medifund.\\nB. Means Testing and Casemix.\\nC. Individual Savings.\\nD. Tax based subsidies and government subvention.\\nAnswer: C. The answer is \"Individual Savings\".\\n---------------------\\nGiven the context information and not prior knowledge, answer the query.\\nQuery: Healthcare System in Singapore consists of?\\nAnswer: ', source_nodes=[NodeWithScore(node=TextNode(id_='536fef67-6a3f-4054-a94a-cc9143599510', embedding=None, metadata={'file_path': '../raw_documents/answers_temp/answers_050.txt', 'file_name': 'answers_050.txt', 'file_type': 'text/plain', 'file_size': 130, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='2b0f7dad-c532-4abd-8c42-f53383a4fc76', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'file_path': '../raw_documents/answers_temp/answers_050.txt', 'file_name': 'answers_050.txt', 'file_type': 'text/plain', 'file_size': 130, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, hash='5b1d1dc729a663e4ccfacc0f18adf0f6644a2a7d2991490fd962d1550c83f2ff'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='6d93c092-b4cc-4b5b-b379-080d777d3908', node_type=<ObjectType.TEXT: '1'>, metadata={'file_path': '../raw_documents/answers_temp/answers_044.txt', 'file_name': 'answers_044.txt', 'file_type': 'text/plain', 'file_size': 164, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, hash='caeb59043b8daa56ed472941882947570abff951f64aa0498672aba5921fac1d'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='859a9958-6f5d-4581-95d0-39edfc950ef5', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='8416454b2fbad3e6122c5151d2b3d1eadf0afde3514ba09374c71e96baf712bc')}, text='Question: The fundamental principle of Singapore healthcare financing is ____________.\\nAnswer: The answer is \"Individual Savings\".', start_char_idx=0, end_char_idx=130, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.4159636550867191), NodeWithScore(node=TextNode(id_='472000ae-a0aa-4464-a200-72fe67a3fbde', embedding=None, metadata={'file_path': '../raw_documents/qna_temp/qna_050.txt', 'file_name': 'qna_050.txt', 'file_type': 'text/plain', 'file_size': 297, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='506fb715-d3b0-4ca7-b7ca-011a1e1a1f0d', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'file_path': '../raw_documents/qna_temp/qna_050.txt', 'file_name': 'qna_050.txt', 'file_type': 'text/plain', 'file_size': 297, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, hash='7461ffa12ff6729003131976b82995b7254ab10f8dc7d79c65988ec9e3b7b631'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='d8232b90-d641-4966-b98f-4ca0821db773', node_type=<ObjectType.TEXT: '1'>, metadata={'file_path': '../raw_documents/qna_temp/qna_044.txt', 'file_name': 'qna_044.txt', 'file_type': 'text/plain', 'file_size': 383, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, hash='cbeb00c29c6130548466697a862fee43ab2be92d84158cc0b69c2f5c7bbe68b1'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='e772e623-cf91-41cd-a516-50acb894eb54', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='a7583b0fd46f98d0118c712632277d81f417b779f8bcc100ab2558dae6317cde')}, text='C1/5\\nQuestion: The fundamental principle of Singapore healthcare financing is ____________.\\nA. The 3’s M. That is Medisave, Medishield, Medifund.\\nB. Means Testing and Casemix.\\nC. Individual Savings.\\nD. Tax based subsidies and government subvention.\\nAnswer: C. The answer is \"Individual Savings\".', start_char_idx=0, end_char_idx=295, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.4126648577998099)], metadata={'536fef67-6a3f-4054-a94a-cc9143599510': {'file_path': '../raw_documents/answers_temp/answers_050.txt', 'file_name': 'answers_050.txt', 'file_type': 'text/plain', 'file_size': 130, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}, '472000ae-a0aa-4464-a200-72fe67a3fbde': {'file_path': '../raw_documents/qna_temp/qna_050.txt', 'file_name': 'qna_050.txt', 'file_type': 'text/plain', 'file_size': 297, 'creation_date': '2024-02-24', 'last_modified_date': '2024-02-24', 'last_accessed_date': '2024-02-24'}})"
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "response = vector_query_engine.query(\"Healthcare System in Singapore consists of?\")\n",
     "response"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d83e2938-61fa-4d02-920d-0ae88a437abc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = vector_query_engine.query(\"what is integrated shield plan\")\n",
+    "response"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -318,7 +306,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "id": "1bb75b04-6a62-43a4-8728-d2e52e49f1c0",
    "metadata": {},
    "outputs": [],
@@ -329,7 +317,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": null,
    "id": "0ed920fb-6456-49ac-8b63-08bd86b5b39c",
    "metadata": {},
    "outputs": [],
@@ -364,7 +352,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "c1a42c35-5f57-423c-8fb7-7d18b3b466b5",
    "metadata": {},
    "outputs": [],
@@ -381,7 +369,14 @@
     "from llama_index.llms.openai import OpenAI\n",
     "from llama_index.core.memory import ChatMemoryBuffer\n",
     "\n",
-    "import time"
+    "import time\n",
+    "\n",
+    "from prompt_engineering import (\n",
+    "            system_content, \n",
+    "            textbook_content, \n",
+    "            winnie_the_pooh_prompt, \n",
+    "            introduction_line\n",
+    ")"
    ]
   },
   {
@@ -394,7 +389,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "d38dc953-b923-4128-86a1-c8c6f69af0ed",
    "metadata": {},
    "outputs": [],
@@ -404,7 +399,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "4c83c613-2cfc-4871-9d07-c82f77a3bd5e",
    "metadata": {},
    "outputs": [],
@@ -414,7 +409,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "0583e9b0-d977-488c-8331-46dfa749924c",
    "metadata": {},
    "outputs": [],
@@ -433,17 +428,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "2159a2b6-494b-41b9-ac54-dd342bfb74ba",
    "metadata": {},
    "outputs": [],
    "source": [
-    "db = chromadb.PersistentClient(path=\"../models/chroma_db_advanced\")"
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db_advanced_corrected\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "1b385644-b46e-4d13-88fa-9f4af39db405",
    "metadata": {},
    "outputs": [],
@@ -453,7 +448,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "93cb53d1-6b8c-4b2d-a839-53501c0d54b2",
    "metadata": {},
    "outputs": [],
@@ -465,7 +460,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "c40d59e1-6d42-41f0-8c9b-70aa026093ae",
    "metadata": {},
    "outputs": [],
@@ -487,20 +482,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "id": "1a506940-c2b4-4d14-ad93-fd451331c582",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "system_content = (\"You are a helpful study assistant. \"\n",
-    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
-    "                  \"You only respond once as 'Assistant'.\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "id": "3f592848-8536-4b4d-b34a-adc32d043432",
    "metadata": {},
    "outputs": [],
@@ -510,7 +492,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "6c7df81a-fd2f-42bf-b09c-46d7750f7252",
    "metadata": {},
    "outputs": [],
@@ -524,7 +506,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "c3106dff-dd6f-47a9-9454-1e61775e7539",
    "metadata": {},
    "outputs": [],
@@ -532,7 +514,7 @@
     "hi_engine = index.as_query_engine(\n",
     "                memory=memory,\n",
     "                system_prompt=system_content,\n",
-    "                similarity_top_k=10,\n",
+    "                similarity_top_k=20,\n",
     "                streaming=True\n",
     ")"
    ]
@@ -547,7 +529,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
    "id": "434f0caf-8b1f-40c6-b9ec-b039cd1ca612",
    "metadata": {},
    "outputs": [],
@@ -563,21 +545,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "id": "a1c83dff-50d1-47b1-b7e9-4fc5cd08e1e8",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "D. To provide for the care of employees\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "res = hi_engine.query(prompt)\n",
-    "print(res)"
+    "response = hi_engine.query(prompt)\n",
+    "for res in response.response_gen:\n",
+    "    print(res, end=\"\")"
    ]
   },
   {
@@ -591,39 +566,34 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "ec53dfcf-d4c0-4d10-a24e-be2004a83656",
+   "id": "91821a22-c1c4-46a6-90f0-c00651afb0f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# query_string = \"tell me more about integrated shield plans\"\n",
+    "# query_string = \"how to use CPF\"\n",
+    "query_string = \"what is MediSave\"\n",
+    "\n",
+    "response = hi_engine.query(query_string)\n",
+    "for res in response.response_gen:\n",
+    "    print(res, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07969feb-2667-4d7d-a769-953082138988",
    "metadata": {},
    "outputs": [],
    "source": []
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
-   "id": "78abaf95-e52d-445c-9d8e-bc51efb20f06",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
-      "To disable this warning, you can either:\n",
-      "\t- Avoid using `tokenizers` before the fork if possible\n",
-      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The correct answer is \"Deductibles apply for all treatments\".\n"
-     ]
-    }
-   ],
-   "source": [
-    "res = chat_engine.chat(prompt)\n",
-    "print(res.response)"
-   ]
+   "execution_count": null,
+   "id": "ec53dfcf-d4c0-4d10-a24e-be2004a83656",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   },
   {
    "cell_type": "code",

notebooks/007_test_hi_content_engine.ipynb

@@ -34,6 +34,12 @@
     "\n",
     "from vision_api import get_transcribed_text\n",
     "from qna_prompting import get_qna_question_tool, evaluate_qna_answer_tool\n",
+    "from prompt_engineering import (\n",
+    "            system_content, \n",
+    "            textbook_content, \n",
+    "            winnie_the_pooh_prompt, \n",
+    "            introduction_line\n",
+    ")\n",
     "\n",
     "import nest_asyncio\n",
     "nest_asyncio.apply()"
@@ -106,18 +112,20 @@
     "\n",
     "        index = VectorStoreIndex(nodes, storage_context=storage_context)\n",
     "    \n",
-    "    memory = ChatMemoryBuffer.from_defaults(token_limit=15000)\n",
+    "    memory = ChatMemoryBuffer.from_defaults(token_limit=100_000)\n",
     "    hi_content_engine = index.as_query_engine(\n",
     "                            memory=memory,\n",
     "                            system_prompt=system_content,\n",
-    "                            similarity_top_k=20,\n",
+    "                            similarity_top_k=10,\n",
+    "                            verbose=True,\n",
     "                            streaming=True\n",
     "    )\n",
     "    hi_textbook_query_description = \"\"\"\n",
-    "        Use this tool to extract content from textbook `Health Insurance 7th Edition`,\n",
+    "        Use this tool to extract content from the query engine,\n",
+    "        which is built by ingesting textbook content from `Health Insurance 7th Edition`,\n",
     "        that has 15 chapters in total. When user wants to learn more about a \n",
     "        particular chapter, this tool will help to assist user to get better\n",
-    "        understanding of the content of the textbook.\n",
+    "        understanding of the content of the textbook. \n",
     "    \"\"\"\n",
     "    \n",
     "    hi_query_tool = QueryEngineTool.from_defaults(\n",
@@ -195,32 +203,10 @@
     "input_files = [\"./raw_documents/HI Chapter Summary Version 1.3.pdf\",\n",
     "               \"./raw_documents/qna.txt\"]\n",
     "embedding_model = \"BAAI/bge-small-en-v1.5\"\n",
-    "persisted_vector_db = \"../models/chroma_db\"\n",
-    "fine_tuned_path = \"local:../models/fine-tuned-embeddings\"\n",
-    "system_content = (\n",
-    "                    \"You are a helpful study assistant. \"\n",
-    "                    \"You do not respond as 'User' or pretend to be 'User'. \"\n",
-    "                    \"You only respond once as 'Assistant'.\"\n",
-    ")\n",
-    "textbook_content = (\n",
-    "                    \"The content of the textbook `Health Insurance 7th Edition` are as follows,\"\n",
-    "                    \"- Chapter 1: Overview Of Healthcare Environment In Singapore\"\n",
-    "                    \"- Chapter 2: Medical Expense Insurance\"\n",
-    "                    \"- Chapter 3: Group Medical Expense Insurance\"\n",
-    "                    \"- Chapter 4: Disability Income Insurance\"\n",
-    "                    \"- Chapter 5: Long-Term Care Insurance \"\n",
-    "                    \"- Chapter 6: Critical Illness Insurance\"\n",
-    "                    \"- Chapter 7: Other Types Of Health Insurance\"\n",
-    "                    \"- Chapter 8: Managed Healthcare\"\n",
-    "                    \"- Chapter 9: Part I Healthcare Financing\"\n",
-    "                    \"- Chapter 9: Part II Healthcare Financing\"\n",
-    "                    \"- Chapter 10: Common Policy Provisions\"\n",
-    "                    \"- Chapter 11: Health Insurance Pricing\"\n",
-    "                    \"- Chapter 12: Health Insurance Underwriting\"\n",
-    "                    \"- Chapter 13: Notice No: MAS 120 Disclosure And Advisory Process - Requirements For Accident And Health Insurance Products\"\n",
-    "                    \"- Chapter 14: Financial Needs Analysis\"\n",
-    "                    \"- Chapter 15: Case Studies\"\n",
-    ")"
+    "persisted_vector_db = \"../models/chroma_db_advanced_corrected\"\n",
+    "\n",
+    "# fine_tuned_path = \"local:../models/fine-tuned-embeddings\"\n",
+    "fine_tuned_path = \"local:../models/fine-tuned-embeddings-advanced\""
    ]
   },
   {
@@ -292,6 +278,14 @@
     ")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a49ed30e-a631-4618-a79e-adab02114d8d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -338,36 +332,39 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "c62e817e-c7c8-4f90-9e32-217fec376565",
+   "id": "66c8881d-fc57-4e95-ad86-110c4818e2fe",
    "metadata": {},
    "outputs": [],
    "source": [
-    "response = hi_content_engine.query(\"can you give me the list of chapters that `Health Insurance 7th Edition` covers\")"
+    "# query_string = \"tell me more about integrated shield plans\"\n",
+    "query_string = \"how to use CPF\"\n",
+    "# query_string = \"what is MediSave\""
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "5902ffd2-2f66-4b89-bf7f-a05e3fdeccaa",
+   "id": "9cbd338b-bee5-4c06-9934-a0e27fd518d3",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "for res in response.response_gen:\n",
-    "    print(res, end=\"\")"
-   ]
+   "source": []
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "0e75453b-85c7-4e1c-8683-6df45a13cacb",
+   "id": "5902ffd2-2f66-4b89-bf7f-a05e3fdeccaa",
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "response = hi_content_engine.query(query_string)\n",
+    "for res in response.response_gen:\n",
+    "    print(res, end=\"\")"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "0b97d90d-5c59-486f-863b-4aaa12ed0ea0",
+   "id": "0e75453b-85c7-4e1c-8683-6df45a13cacb",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -379,7 +376,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "response = agent.stream_chat(\"hihi\", tool_choice=\"auto\")"
+    "response = agent.stream_chat(query_string, tool_choice=\"auto\")\n",
+    "for res in response.response_gen:\n",
+    "    print(res, end=\"\")"
    ]
   },
   {
@@ -388,10 +387,7 @@
    "id": "eff8bb8d-a2d1-428a-9c3d-193389378288",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "for res in response.response_gen:\n",
-    "    print(res, end=\"\")"
-   ]
+   "source": []
   },
   {
    "cell_type": "code",

requirements.txt

@@ -185,6 +185,7 @@ PyMuPDF==1.23.22
 PyMuPDFb==1.23.22
 pyparsing==3.1.1
 pypdf==4.0.1
+PyPDF2==3.0.1
 PyPika==0.48.9
 pyproject_hooks==1.0.0
 python-dateutil==2.8.2