changed to langchain

main.py

@@ -1,45 +1,42 @@
-from haystack.nodes import PreProcessor, PDFToTextConverter, EmbeddingRetriever, TransformersReader
-from haystack.document_stores import InMemoryDocumentStore
-from haystack.pipelines import DocumentSearchPipeline, ExtractiveQAPipeline
+import os
+import sys
+
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import DirectoryLoader, TextLoader
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.indexes import VectorstoreIndexCreator
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS
 import gradio as gr
 
-preprocessor = PreProcessor(
-    clean_empty_lines=True,
-    clean_whitespace=True,
-    clean_header_footer=True,
-    split_by="word",
-    split_length=100,
-    split_respect_sentence_boundary=True,
-    split_overlap=3
-)
-document_store = InMemoryDocumentStore(embedding_dim=384)
-reader = TransformersReader("sentence-transformers/all-MiniLM-L6-v2")
-retriever = EmbeddingRetriever(document_store=document_store, embedding_model="sentence-transformers/all-MiniLM-L6-v2")
-pipeline = ExtractiveQAPipeline(reader, retriever)
-converter = PDFToTextConverter(remove_numeric_tables=True)
+# Enable to cache & reuse the model to disk (for repeated queries on the same data)
+PERSIST = False
 
-def print_answers(results):
-    fields = ["answer", "score"] # "context"
-    answers = results["answers"]
-    filtered_answers = []
-    for ans in answers:
-        filtered_ans = {
-            field: getattr(ans, field) for field in fields if getattr(ans, field) is not None
-        }
-        filtered_answers.append(filtered_ans)
-    return filtered_answers
+query = sys.argv[1]
+embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
 
-def write_pdf(pdf_file):
-    document = converter.convert(file_path=pdf_file.name, meta=None)[0]
-    preprocessed_docs = preprocessor.process(document)
-    document_store.write_documents(preprocessed_docs)
-    document_store.update_embeddings(retriever)
+if PERSIST and os.path.exists("persist"):
+  print("Reusing index...\n")
+  raw_documents = DirectoryLoader("persist").load()
+  text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+  documents = text_splitter.split_documents(raw_documents)
+  vectorstore = FAISS.from_documents(documents, embedding=embeddings)
+  from langchain.indexes.vectorstore import VectorStoreIndexWrapper
+  index = VectorStoreIndexWrapper(vectorstore=vectorstore)
+else:
+  loader = TextLoader('data.txt')
+  # This code can also import folders, including various filetypes like PDFs using the DirectoryLoader.
+  # loader = DirectoryLoader(".", glob="*.txt")
+  if PERSIST:
+    index = VectorstoreIndexCreator(vectorstore_kwargs={"persist_directory":"persist"}).from_loaders([loader])
+  else:
+    index = VectorstoreIndexCreator().from_loaders([loader])
 
-def predict(question, pdf_file):
-    write_pdf(pdf_file)
-    result = pipeline.run(query=question, params={"Retriever": { "top_k": 2 }})
-    answers = print_answers(result)
-    return answers
+chain = RetrievalQA.from_chain_type(
+  llm=,
+  retriever=index.vectorstore.as_retriever(search_kwargs={"k": 5}),
+)
+print(chain.run(query))
 
 interface = gr.Interface(
     fn=predict,

requirements.txt

@@ -1,73 +1,78 @@
-accelerate==0.19.0
-appdirs==1.4.4
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyio==3.7.0
+async-timeout==4.0.2
 attrs==23.1.0
-azure-ai-formrecognizer==3.3.0b1
-azure-common==1.1.28
-azure-core==1.27.0
-backoff==2.2.1
-boilerpy3==1.0.6
-canals==0.2.2
-cattrs==23.1.2
 certifi==2023.5.7
 charset-normalizer==3.1.0
 click==8.1.3
-dill==0.3.6
-docopt==0.6.2
-Events==0.4
-farm-haystack==1.17.1
-filelock==3.12.0
-fsspec==2023.5.0
-generalimport==0.3.1
+contourpy==1.1.0
+cycler==0.11.0
+dataclasses-json==0.5.8
+exceptiongroup==1.1.1
+faiss-cpu==1.7.4
+fastapi==0.97.0
+ffmpy==0.3.0
+filelock==3.12.2
+fonttools==4.40.0
+frozenlist==1.3.3
+fsspec==2023.6.0
+gradio==3.35.2
+gradio_client==0.2.7
+h11==0.14.0
+httpcore==0.17.2
+httpx==0.24.1
 huggingface-hub==0.15.1
 idna==3.4
-inflect==6.0.4
-isodate==0.6.1
+importlib-resources==5.12.0
 Jinja2==3.1.2
-joblib==1.2.0
 jsonschema==4.17.3
+kiwisolver==1.4.4
+langchain==0.0.205
+langchainplus-sdk==0.0.16
+linkify-it-py==2.0.2
+markdown-it-py==2.2.0
 MarkupSafe==2.1.3
-monotonic==1.6
-more-itertools==9.1.0
-mpmath==1.3.0
-msrest==0.7.1
-networkx==3.1
-nltk==3.8.1
-num2words==0.5.12
-numpy==1.24.3
-oauthlib==3.2.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+mypy-extensions==1.0.0
+numexpr==2.8.4
+numpy==1.25.0
+openapi-schema-pydantic==1.2.4
+orjson==3.9.1
 packaging==23.1
 pandas==2.0.2
 Pillow==9.5.0
-posthog==3.0.1
-protobuf==3.20.2
-psutil==5.9.5
-pydantic==1.10.8
+pydantic==1.10.9
+pydub==0.25.1
+Pygments==2.15.1
+pyparsing==3.1.0
 pyrsistent==0.19.3
 python-dateutil==2.8.2
+python-multipart==0.0.6
 pytz==2023.3
 PyYAML==6.0
-quantulum3==0.9.0
-rank-bm25==0.2.2
-regex==2023.6.3
 requests==2.31.0
-requests-cache==0.9.8
-requests-oauthlib==1.3.1
-scikit-learn==1.2.2
-scipy==1.10.1
-sentence-transformers==2.2.2
-sentencepiece==0.1.99
+semantic-version==2.10.0
 six==1.16.0
-sseclient-py==1.7.2
-sympy==1.12
+sniffio==1.3.0
+SQLAlchemy==2.0.16
+starlette==0.27.0
 tenacity==8.2.2
-threadpoolctl==3.1.0
-tiktoken==0.4.0
-tokenizers==0.13.3
-torch==2.0.1
-torchvision==0.15.2
+toolz==0.12.0
 tqdm==4.65.0
-transformers==4.29.1
-typing_extensions==4.5.0
+typing-inspect==0.9.0
+typing_extensions==4.6.3
 tzdata==2023.3
-url-normalize==1.4.3
-urllib3==2.0.2
+uc-micro-py==1.0.2
+urllib3==2.0.3
+uvicorn==0.22.0
+websockets==11.0.3
+yarl==1.9.2
+zipp==3.15.0