File size: 4,163 Bytes
589c9b1 074b93b b403bb0 074b93b 8596e21 c690f92 8596e21 074b93b 8596e21 074b93b b403bb0 ed6e9e8 074b93b c690f92 ed6e9e8 074b93b 589c9b1 074b93b ed6e9e8 074b93b 589c9b1 074b93b 589c9b1 b403bb0 8596e21 b403bb0 8596e21 c690f92 8596e21 b403bb0 c690f92 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
import streamlit as st
#from transformers import pipeline
from pinecone import Pinecone, ServerlessSpec
from sentence_transformers import SentenceTransformer, util
from openai import OpenAI
from datetime import datetime
import pandas as pd
import os
api_key='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'
os.environ["OPENAI_API_KEY"] = api_key
os.environ.get("OPENAI_API_KEY")
bi_encoder = SentenceTransformer('msmarco-distilbert-base-v4')
bi_encoder.max_seq_length = 256 # Truncate long documents to 256 tokens
# Store the index as a variable
INDEX_NAME = 'cl-search-idx'
pc_api_key= '3f916d01-2a69-457d-85eb-966c5d1849a8' #AWS
pc = Pinecone(api_key=pc_api_key)
index = pc.Index(name=INDEX_NAME)
try:
df_log=pd.read_csv('query.csv', index_col=0)
except:
df_log=pd.DataFrame(columns=['query','url', 'result', 'ts'])
def query_from_pinecone(index,namespace, question_embedding, top_k=3):
# get embedding from THE SAME embedder as the documents
return index.query(
vector=question_embedding,
top_k=top_k,
namespace=namespace,
include_metadata=True # gets the metadata (dates, text, etc)
).get('matches')
QUESTION=st.text_area('Ask a question -e.g How to prepare for Verbal section for CAT?') ##' How to prepare for Verbal section ?'
if QUESTION:
question_embedding = bi_encoder.encode(QUESTION, convert_to_tensor=True)
ns='full'
resp= query_from_pinecone(index,ns, question_embedding.tolist(), 3)
if len(resp)>0:
out= resp[0]['metadata']['data']
url= "Matching url "+resp[0]['id']
#+ '\n*************\n'+ resp[1]['metadata']['text'] + '\n*************\n'+ resp[2]['metadata']['text']
system_instructions_text='''
Your task is to extract the answer to a question from a body of text provided to you.
The body of text will be enclosed within the delimiter tags <text> and </text>
For example,
<text> General Preparation Tips for VARC Section:
You need to develop an incessant habit of speed reading.
Start with reading newspapers, editorials, fiction and nonfiction novels and simple passages.
The more you read, the faster you read. Learn the basic grammar concepts like parts of speech, articles,verbs, adjectives, tenses, auxiliary verbs, modifiers, modals etc.
Revise at least 50 new words every day
</text>
Question: What are some tips for preparing for VARC?
Here are some tips for preparing for the VARC section:
1. develop an incessant habit of speed reading
2. Start reading newspapers, editorials, fiction and nonfiction novels
3. Learn basic grammar concepts\n
4. Revise at least 50 new words a day
Question: How many new words are to be learnt in a day?
It is advised that 50 new words are learn every day
Your response should be based on the information contained in the provided text and should not included any other sources.
If you are unable to answer the question from the text provided, please respond " Sorry. I do not have enough information to answer this"
Do repeat the question. Do not make a pointed reference to the text provided. Directly answer the question
'''
client = OpenAI()
content="""
<text>
{}
</text>
""".format(out)
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content":system_instructions_text },
{"role": "user", "content": content},
{"role": "user", "content": "Question:"+QUESTION}
]
)
ans= response.choices[0].message.content
st.write(url)
st.write(ans)
now= str(datetime.utcnow())
df_log.loc[len(df_log)]=[QUESTION,resp[0]['id'],ans,now]
df_log.to_csv('query.csv')
else:
st.write("No matches for query")
now= str(datetime.utcnow())
df_log.loc[len(df_log)]=[QUESTION,'No match','-',now]
df_log.to_csv('query.csv')
|