initial

app.py

@@ -0,0 +1,153 @@
+# -*-coding:utf-8 -*-
+import gradio as gr
+import os
+import json
+from glob import glob
+import requests
+from langchain import FAISS
+from langchain.embeddings import CohereEmbeddings, OpenAIEmbeddings
+from langchain import VectorDBQA
+from langchain.chat_models import ChatOpenAI
+from prompts import MyTemplate
+from build_index.run import process_files
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+)
+
+from langchain.chains.summarize import load_summarize_chain
+from langchain.chains import QAGenerationChain
+
+# Streaming endpoint
+API_URL = "https://api.openai.com/v1/chat/completions"
+cohere_key = '5IRbILAbjTI0VcqTsktBfKsr13Lych9iBAFbLpkj'
+faiss_store = './output/'
+
+
+def process(files, model, max_tokens, openai_api_key, n_sample):
+    """
+    对文档处理进行摘要，构建问题，构建文档索引
+    """
+    model = model[0]
+    os.environ['OPENAI_API_KEY'] = openai_api_key
+    print('Displaying uploading files ')
+    print(glob('/tmp/*'))
+    docs = process_files([i.name for i in files], model, max_tokens)
+    print('Display Faiss index')
+    print(glob('./output/*'))
+    question = get_question(docs, openai_api_key, max_tokens, n_sample)
+    summary = get_summary(docs, openai_api_key, max_tokens, n_sample)
+    return question, summary
+
+
+def get_question(docs, openai_api_key, max_tokens, n_sample=5):
+    q_list = []
+    llm = ChatOpenAI(openai_api_key=openai_api_key, max_tokens=max_tokens, temperature=0)
+    chain = QAGenerationChain.from_llm(llm)
+    print('Generating Question from template')
+    for i in range(n_sample):
+        qa = chain.run(docs[i].page_content)[0]
+        print(qa)
+        q_list.append(f"问题{i+1}: {qa['question']}" )
+    return '\n'.join(q_list)
+
+
+def get_summary(docs, openai_api_key, max_tokens, n_sample=5):
+    llm = ChatOpenAI(openai_api_key=openai_api_key, max_tokens=max_tokens)
+    chain = load_summarize_chain(llm, chain_type="map_reduce")
+    print('Generating Summary from tempalte')
+    summary = chain.run(docs[:n_sample])
+    print(summary)
+    return summary
+
+
+def predict(inputs, openai_api_key, max_tokens, model, chat_counter, chatbot=[], history=[]):
+    model = model[0]
+    print(f"chat_counter - {chat_counter}")
+    print(f'Histroy - {history}')  # History: Original Input and Output in flatten list
+    print(f'chatbot - {chatbot}')  # Chat Bot： 上一轮回复的[[user, AI]]
+
+    history.append(inputs)
+    print(f'loading faiss store from {faiss_store}')
+    if model == 'openai':
+        docsearch = FAISS.load_local(faiss_store, OpenAIEmbeddings(openai_api_key=openai_api_key))
+    else:
+        docsearch = FAISS.load_local(faiss_store, CohereEmbeddings(cohere_api_key=cohere_key))
+    # 构建模板
+    llm = ChatOpenAI(openai_api_key=openai_api_key, max_tokens=max_tokens)
+    messages_combine = [
+        SystemMessagePromptTemplate.from_template(MyTemplate['chat_combine_template']),
+        HumanMessagePromptTemplate.from_template("{question}")
+    ]
+    p_chat_combine = ChatPromptTemplate.from_messages(messages_combine)
+    messages_reduce = [
+        SystemMessagePromptTemplate.from_template(MyTemplate['chat_reduce_template']),
+        HumanMessagePromptTemplate.from_template("{question}")
+    ]
+    p_chat_reduce = ChatPromptTemplate.from_messages(messages_reduce)
+    chain = VectorDBQA.from_chain_type(llm=llm, chain_type="map_reduce", vectorstore=docsearch,
+                                       k=4,
+                                       chain_type_kwargs={"question_prompt": p_chat_reduce,
+                                                          "combine_prompt": p_chat_combine}
+                                       )
+    result = chain({"query": inputs})
+    print(result)
+    result = result['result']
+    # 生成返回值
+    history.append(result)
+    chat = [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2)]
+    chat_counter += 1
+    yield chat, history, chat_counter
+
+
+def reset_textbox():
+    return gr.update(value='')
+
+
+with gr.Blocks(css="""#col_container {width: 1000px; margin-left: auto; margin-right: auto;}
+                #chatbot {height: 520px; overflow: auto;}""") as demo:
+    gr.HTML("""<h1 align="center">🚀Your Doc Reader🚀</h1>""")
+    with gr.Column(elem_id="col_container"):
+        openai_api_key = gr.Textbox(type='password', label="输入 API Key")
+
+        with gr.Accordion("Parameters", open=True):
+            with gr.Row():
+                max_tokens = gr.Slider(minimum=100, maximum=2000, value=1000, step=100, interactive=True,
+                                       label="字数")
+                model = gr.CheckboxGroup(["cohere", "openai"])
+                chat_counter = gr.Number(value=0, precision=0, label='对话轮数')
+                n_sample = gr.Slider(minimum=3, maximum=5, value=3, step=1, interactive=True,
+                                     label="问题数")
+
+        # 输入文件，进行摘要和��题生成
+        with gr.Row():
+            with gr.Column():
+                files = gr.File(file_count="multiple", file_types=[".pdf"], label='上传pdf文件')
+                run = gr.Button('研报解读')
+
+            with gr.Column():
+                summary = gr.Textbox(type='text', label="本文摘要")
+                question = gr.Textbox(type='text', label='提问问题')
+
+        chatbot = gr.Chatbot(elem_id='chatbot')
+        inputs = gr.Textbox(placeholder="这篇文档是关于什么的", label="针对文档你有哪些问题？")
+        state = gr.State([])
+
+        with gr.Row():
+            clear = gr.Button("清空")
+            start = gr.Button("提问")
+
+    run.click(process, [files, openai_api_key, max_tokens, model, n_sample], [question, summary])
+    inputs.submit(predict,
+                  [inputs, openai_api_key, max_tokens, model, chat_counter, chatbot, state],
+                  [chatbot, state, chat_counter], )
+    start.click(predict,
+              [inputs, openai_api_key, max_tokens, model, chat_counter, chatbot, state],
+              [chatbot, state, chat_counter], )
+
+    # 每次对话结束都重置对话
+    clear.click(reset_textbox, [], [inputs], queue=False)
+    inputs.submit(reset_textbox, [], [inputs])
+
+    demo.queue().launch(debug=True)

build_index/__init__.py

@@ -0,0 +1 @@
+# -*-coding:utf-8 -*-

build_index/base.py

@@ -0,0 +1,88 @@
+# -*-coding:utf-8 -*-
+"""
+    Base Reader and Document
+"""
+import os
+from dataclasses import dataclass
+from dataclasses_json import dataclass_json
+from typing import Any, Dict, List, Optional
+from glob import glob
+from build_index.parser import ParserFactory
+from langchain.docstore.document import Document as LCDocument
+
+
+@dataclass_json
+@dataclass
+class Document:
+    text: str = None
+    doc_id: Optional[str] = None
+    embedding: Optional[List[float]] = None
+    extra_info: Optional[Dict[str, Any]] = None
+
+    def get_text(self):
+        return self.text
+
+    def get_doc_id(self):
+        return self.doc_id
+
+    def get_embedding(self):
+        return self.embedding
+
+    @property
+    def extra_info_str(self) -> Optional[str]:
+        """Extra info string."""
+        if self.extra_info is None:
+            return None
+
+        return "\n".join([f"{k}: {str(v)}" for k, v in self.extra_info.items()])
+
+    def __post_init__(self):
+        #字段检查
+        assert self.text is not None, 'Text Field can not be None'
+
+    def to_langchain_format(self):
+        """Convert struct to LangChain document format."""
+        metadata = self.extra_info or {}
+        return LCDocument(page_content=self.text, metadata=metadata)
+
+
+class FileReader(object):
+    """
+    Load file from ./data_dir
+    """
+    def __init__(self, data_dir=None, folder_name=None, input_files=None, has_meta=True):
+        self.data_dir = data_dir
+        self.has_meta = has_meta
+
+        if input_files:
+            self.input_files = input_files
+        else:
+            # get all file in data_dir
+            ##TODO: 暂不支持data下recursive dir
+            dir = os.path.join(data_dir, folder_name, '*')
+            self.input_files = glob(dir)
+            print(f'{len(self.input_files)} files in {dir}')
+            print(self.input_files)
+
+    def load_data(self, concatenate=False) -> List[Document]:
+        data_list = []
+        metadata_list = []
+        for file in self.input_files:
+            parser = ParserFactory['pdf']
+            if parser is None:
+                raise ValueError(f"{file} format doesn't match any sufix supported")
+            try:
+                data, meta = parser.parse_file(file)
+            except Exception as e:
+                print(f'{file} parse failed. error = {e}')
+                continue
+            data_list.append(data)
+            if self.has_meta:
+                metadata_list.append(meta)
+
+        if concatenate:
+            return [Document("\n".join(data_list))]
+        elif self.has_meta:
+            return [Document(d, extra_info=m) for d, m in zip(data_list, metadata_list)]
+        else:
+            return [Document(d) for d in data_list]

build_index/doc2vec.py

@@ -0,0 +1,52 @@
+# -*-coding:utf-8 -*-
+import os
+from tqdm import tqdm
+from langchain.vectorstores import FAISS
+from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings, CohereEmbeddings
+from retry import retry
+from key import CoherenceKey, OpenaiKey
+
+# Output Directory for FAISS Index data
+OUTPUT_DIR = './output/'
+
+
+@retry(tries=10, delay=60)
+def store_add_texts_with_retry(store, i):
+    store.add_texts([i.page_content], metadatas=[i.metadata])
+
+
+def doc2vec(docs, model, folder_name=None):
+    if folder_name:
+        dir = os.path.join(OUTPUT_DIR, folder_name)
+    else:
+        dir = OUTPUT_DIR
+    # use first document to init db, 1个1个文件处理避免中间出现问题需要重头尝试
+    print(f'Building faiss Index from {len(docs)} docs')
+    docs_test = [docs[0]]
+    docs.pop(0)
+    index = 0
+
+    print(f'Dumping FAISS to {dir}')
+    if model =='openai':
+        key = os.getenv('OPENAI_API_KEY')
+        db = FAISS.from_documents(docs_test, OpenAIEmbeddings(openai_api_key=key))
+    elif model =='mpnet':
+        db = FAISS.from_documents(docs_test, HuggingFaceEmbeddings())
+    elif model =='cohere':
+        db = FAISS.from_documents(docs_test, CohereEmbeddings(cohere_api_key=CoherenceKey))
+    else:
+        raise ValueError(f'Embedding Model {model} not supported')
+
+    for doc in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs),
+                  bar_format='{l_bar}{bar}| Time Left: {remaining}'):
+        try:
+            store_add_texts_with_retry(db, doc)
+        except Exception as e:
+            print(e)
+            print("Error on ", doc)
+            print("Saving progress")
+            print(f"stopped at {index} out of {len(docs)}")
+            db.save_local(dir)
+            break
+        index += 1
+    db.save_local(dir)

build_index/parser/__init__.py

@@ -0,0 +1,7 @@
+# -*-coding:utf-8 -*-
+from .html_parser import *
+from .pdf_parser import *
+ParserFactory = {
+    'html': HTMLParser(),
+    'pdf': PDFParser()
+}

build_index/parser/base.py

@@ -0,0 +1,39 @@
+# -*-coding:utf-8 -*-
+"""
+    解析不同格式的文档
+"""
+import re
+import abc
+from typing import Union, List
+
+
+class BaseParser(abc.ABC):
+
+    def __init__(self, config=None):
+        self.config = config
+
+
+    @staticmethod
+    def full2half(text):
+        s = ''
+        for c in text:
+            num = ord(c)
+            if num == 0x3000:
+                num = 0x20
+            elif 0xFF01 <= num <= 0xFF5E:
+                num = num - 0xFEE0
+            s += chr(num)
+        return s
+
+    @staticmethod
+    def remove_dup_space(text):
+        text = re.sub(r'\s{1,}', '', text, flags=re.MULTILINE|re.DOTALL)  # Extra Spacee Remove
+        return text
+
+    @staticmethod
+    def remove_empty_line(text):
+        text = re.sub(r'\n', '', text, flags=re.MULTILINE|re.DOTALL)  # Extra Spacee Remove
+        return text
+
+    def parse_file(self, file):
+        raise NotImplementedError

build_index/parser/html_parser.py

@@ -0,0 +1,48 @@
+# -*-coding:utf-8 -*-
+
+import re
+from unstructured.partition.html import partition_html
+from unstructured.staging.base import convert_to_isd
+from unstructured.cleaners.core import clean
+from build_index.parser.base import BaseParser
+
+
+class HTMLParser(BaseParser):
+    def parse_file(self, file):
+        with open(file, "r", encoding="utf-8") as fp:
+            elements = partition_html(file=fp)
+            isd = convert_to_isd(elements)
+
+        for isd_el in isd:
+            isd_el['text'] = isd_el['text'].encode("ascii", "ignore").decode()
+            isd_el['text'] = self.remove_dup_space(isd_el['text'])
+            isd_el['text'] = self.remove_empty_line(isd_el['text'])
+            clean(isd_el['text'], extra_whitespace=True, dashes=True, bullets=True, trailing_punctuation=True )
+
+        # Creating a list of all the indexes of isd_el['type'] = 'Title'
+        title_indexes = [i for i, isd_el in enumerate(isd) if isd_el['type'] == 'Title']
+
+        # Creating 'Chunks' - List of lists of strings
+        # each list starting with with isd_el['type'] = 'Title' and all the data till the next 'Title'
+        # Each Chunk can be thought of as an individual set of data, which can be sent to the model
+        # Where Each Title is grouped together with the data under it
+
+        Chunks = [[]]
+        final_chunks = list(list())
+
+        for i, isd_el in enumerate(isd):
+            if i in title_indexes:
+                Chunks.append([])
+            Chunks[-1].append(isd_el['text'])
+
+        # Removing all the chunks with sum of lenth of all the strings in the chunk < 25 #TODO: This value can be an user defined variable
+        for chunk in Chunks:
+            # sum of lenth of all the strings in the chunk
+            sum = 0
+            sum += len(str(chunk))
+            if sum < 25:
+                Chunks.remove(chunk)
+            else :
+                # appending all the approved chunks to final_chunks as a single string
+                final_chunks.append(" ".join([str(item) for item in chunk]))
+        return final_chunks

build_index/parser/pdf_parser.py

@@ -0,0 +1,27 @@
+# -*-coding:utf-8 -*-
+import PyPDF2
+from build_index.parser.base import BaseParser
+
+
+class PDFParser(BaseParser):
+    def header_remove(self):
+        # 删除研报的页头
+        pass
+
+    def footnote_remove(self):
+        # 删除研报的页脚
+        pass
+
+    def parse_file(self, file):
+        # store pages of
+        text_list = []
+
+        with open(file, "rb") as fp:
+            pdf = PyPDF2.PdfReader(fp)
+            num_pages = len(pdf.pages)
+            for page in range(num_pages):
+                page_text = pdf.pages[page].extract_text()
+                text_list.append(page_text)
+        text = '\n'.join(text_list)
+        metadata = {'source': file, 'pages': num_pages}
+        return text, metadata

build_index/pricing.py

@@ -0,0 +1,23 @@
+# -*-coding:utf-8 -*-
+import tiktoken
+
+
+def num_tokens_from_string(string: str, encoding_name: str):
+    # Function to convert string to tokens and estimate user cost.
+    encoding = tiktoken.get_encoding(encoding_name)
+    num_tokens = len(encoding.encode(string))
+    total_price = ((num_tokens / 1000) * 0.0004)
+    return num_tokens, total_price
+
+
+def check_price(docs):
+    docs_content = ""
+    for doc in docs:
+        docs_content += doc.page_content
+
+    tokens, total_price = num_tokens_from_string(string=docs_content, encoding_name="cl100k_base")
+
+    print(f"Number of Tokens = {format(tokens, ',d')}")
+    print(f"Approx Cost = ${format(total_price, ',.2f')}")
+    user_input = input("Price Okay? (Y/N) \n").upper() == 'Y'
+    return user_input

build_index/process.py

@@ -0,0 +1,78 @@
+# -*-coding:utf-8 -*-
+"""
+    Split Document into chunks
+"""
+
+import re
+import tiktoken
+
+from typing import List
+from build_index.base import Document
+from math import ceil
+
+
+def separate_header_and_body(text):
+    header_pattern = r"^(.*?\n){3}"
+    match = re.match(header_pattern, text)
+    header = match.group(0)
+    body = text[len(header):]
+    return header, body
+
+
+def group_documents(documents: List[Document], min_tokens: int, max_tokens: int) -> List[Document]:
+    docs = []
+    current_group = None
+
+    for doc in documents:
+        doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
+
+        if current_group is None:
+            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
+                                     extra_info=doc.extra_info)
+        elif len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and doc_len >= min_tokens:
+            current_group.text += " " + doc.text
+        else:
+            docs.append(current_group)
+            current_group = Document(text=doc.text, doc_id=doc.doc_id, embedding=doc.embedding,
+                                     extra_info=doc.extra_info)
+
+    if current_group is not None:
+        docs.append(current_group)
+
+    return docs
+
+
+def split_documents(documents: List[Document], max_tokens: int) -> List[Document]:
+    docs = []
+    for doc in documents:
+        token_length = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
+        if token_length <= max_tokens:
+            docs.append(doc)
+        else:
+            header, body = separate_header_and_body(doc.text)
+            num_body_parts = ceil(token_length / max_tokens)
+            part_length = ceil(len(body) / num_body_parts)
+            body_parts = [body[i:i + part_length] for i in range(0, len(body), part_length)]
+            for i, body_part in enumerate(body_parts):
+                new_doc = Document(text=header + body_part.strip(),
+                                   doc_id=f"{doc.doc_id}-{i}",
+                                   embedding=doc.embedding,
+                                   extra_info=doc.extra_info)
+                docs.append(new_doc)
+    return docs
+
+
+def group_split(documents: List[Document], max_tokens: int = 2000, min_tokens: int = 150, token_check: bool = True):
+    if token_check == False:
+        return documents
+    print("Grouping small documents")
+    try:
+        documents = group_documents(documents=documents, min_tokens=min_tokens, max_tokens=max_tokens)
+    except:
+        print("Grouping failed, try running without token_check")
+    print("Separating large documents")
+    try:
+        documents = split_documents(documents=documents, max_tokens=max_tokens)
+    except:
+        print("Grouping failed, try running without token_check")
+    return documents

build_index/run.py

@@ -0,0 +1,56 @@
+# -*-coding:utf-8 -*-
+"""
+    Build Faiss Index for all document in ./data
+"""
+
+import argparse
+from build_index.base import FileReader, Document
+from build_index.process import split_documents
+from build_index.pricing import check_price
+from build_index.doc2vec import doc2vec
+
+
+def process_directory(directory, folder_name, model, max_tokens):
+    reader = FileReader(directory, folder_name)
+    docs = reader.load_data()
+    print(f'Before chunking {len(docs)} docs')
+    new_docs = split_documents(documents=docs, max_tokens=max_tokens)
+    print(f'After chunking {len(new_docs)} docs')
+    docs = [Document.to_langchain_format(doc) for doc in new_docs]
+
+    doc2vec(docs, model, folder_name)
+    return docs
+
+
+def process_files(files, model, max_tokens):
+    reader = FileReader(input_files=files)
+    docs = reader.load_data()
+    print(f'Before chunking {len(docs)} docs')
+    new_docs = split_documents(documents=docs, max_tokens=max_tokens)
+    print(f'After chunking {len(new_docs)} docs')
+    docs = [Document.to_langchain_format(doc) for doc in new_docs]
+
+    doc2vec(docs, model)
+    return docs
+
+
+if __name__ =='__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory", type=str, deafult='./data')
+    parser.add_argument("folder", type=str, default='半导体')
+    parser.add_argument("model", type=str, default='mpnet')
+    parser.add_argument('max_tokens', type=int, default=514)
+    args = parser.parse_args()
+
+    process_directory(args.directory, args.folder, args.model, args.max_token)
+
+    process_directory('./data', '半导体', 'mpnet', 514)
+    process_directory('./data', '数字经济', 'cohere', 514)
+    process_directory('./data', '数字经济', 'cohere', 514)
+    process_directory('./data', '两会', 'cohere', 514)
+
+    from glob import glob
+    files = glob('./data/两会/*.pdf')
+    files = files[:3]
+    process_files(files[:3], 'cohere', 514)

build_index/unit_test/__init__.py

@@ -0,0 +1 @@
+# -*-coding:utf-8 -*-

build_index/unit_test/test_faiss.py

@@ -0,0 +1,33 @@
+# -*-coding:utf-8 -*-
+
+from langchain.vectorstores import FAISS
+from langchain.embeddings import HuggingFaceEmbeddings, CohereEmbeddings
+from key import CoherenceKey
+
+## MPNET
+model = HuggingFaceEmbeddings()
+db = FAISS.load_local('./output/半导体', model)
+
+docs = db.similarity_search('东吴证券观点')
+print(docs[0].page_content)
+
+
+docs = db.similarity_search('德邦证券')
+print(docs[0].page_content)
+
+
+## Coherence
+model = CohereEmbeddings(cohere_api_key=CoherenceKey)
+db = FAISS.load_local('./output/半导体', model)
+
+docs = db.similarity_search('半导体指数行情')
+print(docs[0].page_content)
+
+
+docs = db.similarity_search('关于行业光刻胶相关新闻')
+print(docs[0].page_content)
+
+
+
+docs = db.similarity_search('2023年GDP增速预测')
+print(docs[0].page_content)

build_index/unit_test/test_loader.py

@@ -0,0 +1 @@
+# -*-coding:utf-8 -*-

key.py

@@ -0,0 +1,4 @@
+# -*-coding:utf-8 -*-
+CoherenceKey = '5IRbILAbjTI0VcqTsktBfKsr13Lych9iBAFbLpkj'
+#OpenaiKey='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'
+OpenaiKey='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'

prompts/__init__.py

@@ -0,0 +1,22 @@
+# -*-coding:utf-8 -*-
+
+# load prompt template
+with open("prompts/combine_prompt.txt", "r") as f:
+    template = f.read()
+
+with open("prompts/combine_prompt_hist.txt", "r") as f:
+    template_hist = f.read()
+
+with open("prompts/chat_combine_prompt.txt", "r") as f:
+    chat_combine_template = f.read()
+
+with open("prompts/chat_reduce_prompt.txt", "r") as f:
+    chat_reduce_template = f.read()
+
+
+MyTemplate ={
+    'chat_reduce_template': chat_reduce_template,
+    'chat_combine_template': chat_combine_template,
+    'template_hist': template_hist,
+    'template':template
+}

prompts/chat_combine_prompt.txt

@@ -0,0 +1,4 @@
+You are a DocsGPT, friendly and helpful AI assistant by TianHong Asset Managementthat provides help with documents and financial news. You give thorough answers with detail number and illustrated examples if possible.
+Use the following pieces of context to help answer the users question, always answer in chinese.
+----------------
+{summaries}

prompts/chat_reduce_prompt.txt

@@ -0,0 +1,3 @@
+Use the following portion of a long document to see if any of the text is relevant to answer the question.
+{context}
+Provide all relevant text to the question verbatim. Summarize if needed, Answer in Chinese and illustratively. If nothing relevant return "-".

prompts/combine_prompt.txt

@@ -0,0 +1,25 @@
+You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples if possible.
+
+QUESTION: How to merge tables in pandas?
+=========
+Content: pandas provides various facilities for easily combining together Series or DataFrame with various kinds of set logic for the indexes and relational algebra functionality in the case of join / merge-type operations.
+Source: 28-pl
+Content: pandas provides a single function, merge(), as the entry point for all standard database join operations between DataFrame or named Series objects: \n\npandas.merge(left, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None)
+Source: 30-pl
+=========
+FINAL ANSWER: To merge two tables in pandas, you can use the pd.merge() function. The basic syntax is: \n\npd.merge(left, right, on, how) \n\nwhere left and right are the two tables to merge, on is the column to merge on, and how is the type of merge to perform. \n\nFor example, to merge the two tables df1 and df2 on the column 'id', you can use: \n\npd.merge(df1, df2, on='id', how='inner')
+SOURCES: 28-pl 30-pl
+
+QUESTION: How are you?
+=========
+CONTENT:
+SOURCE:
+=========
+FINAL ANSWER: I am fine, thank you. How are you?
+SOURCES:
+
+QUESTION: {{ question }}
+=========
+{{ summaries }}
+=========
+FINAL ANSWER:

prompts/combine_prompt_hist.txt

@@ -0,0 +1,33 @@
+You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples if possible.
+
+QUESTION: How to merge tables in pandas?
+=========
+Content: pandas provides various facilities for easily combining together Series or DataFrame with various kinds of set logic for the indexes and relational algebra functionality in the case of join / merge-type operations.
+Source: 28-pl
+Content: pandas provides a single function, merge(), as the entry point for all standard database join operations between DataFrame or named Series objects: \n\npandas.merge(left, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None)
+Source: 30-pl
+=========
+FINAL ANSWER: To merge two tables in pandas, you can use the pd.merge() function. The basic syntax is: \n\npd.merge(left, right, on, how) \n\nwhere left and right are the two tables to merge, on is the column to merge on, and how is the type of merge to perform. \n\nFor example, to merge the two tables df1 and df2 on the column 'id', you can use: \n\npd.merge(df1, df2, on='id', how='inner')
+SOURCES: 28-pl 30-pl
+
+QUESTION: How are you?
+=========
+CONTENT:
+SOURCE:
+=========
+FINAL ANSWER: I am fine, thank you. How are you?
+SOURCES:
+
+QUESTION: {{ historyquestion }}
+=========
+CONTENT:
+SOURCE:
+=========
+FINAL ANSWER: {{ historyanswer }}
+SOURCES:
+
+QUESTION: {{ question }}
+=========
+{{ summaries }}
+=========
+FINAL ANSWER:

prompts/question_prompt.txt

@@ -0,0 +1,4 @@
+Use the following portion of a long document to see if any of the text is relevant to answer the question.
+{{ context }}
+Question: {{ question }}
+Provide all relevant text to the question verbatim. Summarize if needed. If nothing relevant return "-".

prompts/style.py

@@ -0,0 +1,14 @@
+# -*-coding:utf-8 -*-
+
+Prompt = {
+    # 文章润色: 修改拼写，语法，提高可读性
+    'Better': 'As a writing improvement assistant, your task is to improve the spelling, grammar, clarity, concision, and overall readability of the text provided, while breaking down long sentences, reducing repetition, and providing suggestions for improvement. Please provide only the corrected Chinese version of the text and avoid including explanations. Please begin by editing the following text: {Text}',
+    # 小红书画风: 适合宣传，营销场景
+    'RedBook':'Please edit the following passage in Chinese using the Xiaohongshu style, which is characterized by captivating headlines, the inclusion of emoticons in each paragraph, and the addition of relevant tags at the end. Be sure to maintain the original meaning of the text. Please begin by editing the following text: {Text}',
+    # 去口语化: 口语转正式文本，用户访谈转正式文本
+    'Decolloquialism': 'Using concise and clear language, please edit the following passage to improve its logical flow, eliminate any typographical errors and respond in Chinese. Be sure to maintain the original meaning of the text. Please begin by editing the following text: {Text}',
+    # 文本缩写摘要
+    'Summarize':'Summarize the following text into 100 words, making it easy to read and comprehend. The summary should be concise, clear, and capture the main points of the text. Avoid using complex sentence structures or technical jargon. Please begin by editing the following text: {Text}',
+    # 幽默搞笑画风
+    'Humor': 'Rewrite the article with a stylistic flair that retains the original semantic meaning while making it more humorous and witty. Respond in Chinese. Please begin by editing the following text: {Text} '
+}

requirements.txt

@@ -0,0 +1,10 @@
+openai==0.27.2
+gradio==3.21.0
+langchain==0.0.113
+unstructured==0.4.11
+PyPDF2==3.0.1
+tiktoken==0.1.2
+dataclasses_json==0.5.7
+retry==0.9.2
+cohere==3.4.0
+faiss-cpu==1.7.3