UPDATE: loadPDF

app.py

@@ -2,6 +2,7 @@ import io
 import tempfile
 import jwt
 import base64
+import json
 from click import option
 from jwt import ExpiredSignatureError, InvalidTokenError
 from starlette import status
@@ -251,18 +252,34 @@ async def newChatbot(chatbotName: str, username: str):
 @app.post("/loadPDF")
 async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
     source = pdf.filename
+    username, chatbotName = vectorstore.split("$")[1], vectorstore.split("$")[2]
     pdf = await pdf.read()
     with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
         temp_file.write(pdf)
         temp_file_path = temp_file.name
     text = extractTextFromPdf(temp_file_path)
     os.remove(temp_file_path)
+    dct = {
+        "output": text
+    }
+    dct = json.dumps(dct, indent = 1)
+    fileName = createDataSourceName(sourceName = source)
+    response = supabase.storage.from_("ConversAI").upload(file=dct, path=f"{fileName}_data.json")
+    response = (
+        supabase.table("ConversAI_ChatbotDataSources")
+        .insert({"username": username, 
+                 "chatbotName": chatbotName, 
+                 "dataSourceName": fileName, 
+                 "sourceEndpoint": "\loadPDF", 
+                 "sourceContentURL": os.path.join(os.environ["SUPABASE_PUBLIC_BASE_URL"], f"{fileName}_data.json")})
+        .execute()
+    )
     return {
-        "output": text,
-        "source": source
+        "output": "SUCCESS"
     }
 
 
+
 @app.post("/loadImagePDF")
 async def returnText(pdf: UploadFile = File(...)):
     source = pdf.filename

functions.py

@@ -347,4 +347,15 @@ def extractTextFromUrl(url):
 def extractTextFromUrlList(urls):
     with ThreadPoolExecutor() as executor:
         texts = list(executor.map(extractTextFromUrl, urls))
-    return {x: y for x, y in zip(urls, texts)}
+    return {x: y for x, y in zip(urls, texts)}
+
+
+def createDataSourceName(sourceName):
+  sources = [x["dataSourceName"] for x in client.table("ConversAI_ChatbotDataSources").select("dataSourceName").execute().data]
+  if sourceName not in sources:
+    return sourceName
+  else:
+    i = 1
+    while True:
+      sourceName = sourceName + "-" + str(i)
+      return createDataSourceName(sourceName)

secrets.env

@@ -6,4 +6,5 @@ QDRANT_API_KEY=k0V8kKNulQdRLukhYy03kJcncctoDImbiPHgmvaEEsup8MwTjqgT0w
 COHERE_API_KEY=lCu3rZEjcUPAt0RsdQpQlGtgYp1uKAmuNIBdjFKq
 NLTK_DATA=/app/nltk_data
 LANGCHAIN_API_KEY=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
-LANGCHAIN_TRACING_V2=true
+LANGCHAIN_TRACING_V2=true
+SUPABASE_PUBLIC_BASE_URL=https://lvuhhlrkcuexzqtsbqyu.supabase.co/storage/v1/object/public/ConversAI