Integrated speech transcription

app.py

@@ -6,10 +6,10 @@ from fastapi import FastAPI, File, UploadFile
 from pydantic import BaseModel
 from fastapi.middleware.cors import CORSMiddleware
 from langchain_community.document_loaders import UnstructuredURLLoader
+from src.api.speech_api import speech_translator_router
 
+app = FastAPI(title="ConversAI", root_path="/api/v1")
 
-
-app = FastAPI(title = "ConversAI", root_path = "/api/v1")
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -18,29 +18,33 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+app.include_router(speech_translator_router, prefix="/speech")
+
+
 @app.post("/signup")
 async def signup(username: str, password: str):
-    response = createUser(username = username, password = password)
+    response = createUser(username=username, password=password)
     return response
 
 
 @app.post("/login")
 async def login(username: str, password: str):
-    response = matchPassword(username = username, password = password)
+    response = matchPassword(username=username, password=password)
     return response
 
 
 @app.post("/newChatbot")
 async def newChatbot(chatbotName: str, username: str):
-    currentBotCount = len(listTables(username = username)["output"])
-    limit = client.table("ConversAI_UserConfig").select("chatbotLimit").eq("username", username).execute().data[0]["chatbotLimit"]
+    currentBotCount = len(listTables(username=username)["output"])
+    limit = client.table("ConversAI_UserConfig").select("chatbotLimit").eq("username", username).execute().data[0][
+        "chatbotLimit"]
     if currentBotCount >= int(limit):
         return {
             "output": "CHATBOT LIMIT EXCEEDED"
         }
     client.table("ConversAI_ChatbotInfo").insert({"username": username, "chatbotname": chatbotName}).execute()
     chatbotName = f"convai-{username}-{chatbotName}"
-    return createTable(tablename = chatbotName)
+    return createTable(tablename=chatbotName)
 
 
 @app.post("/addPDF")
@@ -53,11 +57,13 @@ async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
     username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
     currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
-    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+        "tokenLimit"]
     newCount = currentCount + len(text)
     if newCount < int(limit):
-        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
-        return addDocuments(text = text, vectorstore = vectorstore)
+        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+            "chatbotname", chatbotname).execute()
+        return addDocuments(text=text, vectorstore=vectorstore)
     else:
         return {
             "output": "DOCUMENT EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
@@ -67,7 +73,7 @@ async def addPDFData(vectorstore: str, pdf: UploadFile = File(...)):
 @app.post("/scanAndReturnText")
 async def returnText(pdf: UploadFile = File(...)):
     pdf = await pdf.read()
-    text = getTextFromImagePDF(pdfBytes = pdf)
+    text = getTextFromImagePDF(pdfBytes=pdf)
     return text
 
 
@@ -77,10 +83,12 @@ async def addText(vectorstore: str, text: str):
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
     currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
     newCount = currentCount + len(text)
-    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+        "tokenLimit"]
     if newCount < int(limit):
-        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
-        return addDocuments(text = text, vectorstore = vectorstore)
+        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+            "chatbotname", chatbotname).execute()
+        return addDocuments(text=text, vectorstore=vectorstore)
     else:
         return {
             "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
@@ -100,10 +108,12 @@ async def addText(addQaPair: AddQAPair):
     currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
     qa = f"QUESTION: {addQaPair.question}\tANSWER: {addQaPair.answer}"
     newCount = currentCount + len(qa)
-    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+        "tokenLimit"]
     if newCount < int(limit):
-        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
-        return addDocuments(text = qa, vectorstore = addQaPair.vectorstore)
+        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+            "chatbotname", chatbotname).execute()
+        return addDocuments(text=qa, vectorstore=addQaPair.vectorstore)
     else:
         return {
             "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
@@ -115,20 +125,24 @@ async def addWebsite(vectorstore: str, websiteUrls: list[str]):
     urls = websiteUrls
     loader = UnstructuredURLLoader(urls=urls)
     docs = loader.load()
-    text = "\n\n".join([f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
+    text = "\n\n".join(
+        [f"Metadata:\n{docs[doc].metadata} \nPage Content:\n {docs[doc].page_content}" for doc in range(len(docs))])
     username, chatbotname = vectorstore.split("-")[1], vectorstore.split("-")[2]
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
     currentCount = df[(df["username"] == username) & (df["chatbotname"] == chatbotname)]["charactercount"].iloc[0]
     newCount = currentCount + len(text)
-    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0]["tokenLimit"]
+    limit = client.table("ConversAI_UserConfig").select("tokenLimit").eq("username", username).execute().data[0][
+        "tokenLimit"]
     if newCount < int(limit):
-        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq("chatbotname", chatbotname).execute()
-        return addDocuments(text = text, vectorstore = vectorstore)
+        client.table("ConversAI_ChatbotInfo").update({"charactercount": str(newCount)}).eq("username", username).eq(
+            "chatbotname", chatbotname).execute()
+        return addDocuments(text=text, vectorstore=vectorstore)
     else:
         return {
             "output": "WEBSITE EXCEEDING LIMITS, PLEASE TRY WITH A SMALLER DOCUMENT."
         }
 
+
 @app.post("/answerQuery")
 async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-70b-8192"):
     return answerQuery(query=query, vectorstore=vectorstore, llmModel=llmModel)
@@ -140,15 +154,18 @@ async def delete(chatbotName: str):
     client.table('ConversAI_ChatbotInfo').delete().eq('username', username).eq('chatbotname', chatbotName).execute()
     return deleteTable(tableName=chatbotName)
 
+
 @app.post("/listChatbots")
 async def delete(username: str):
     return listTables(username=username)
 
+
 @app.post("/getLinks")
 async def crawlUrl(baseUrl: str):
     return {
         "urls": getLinks(url=baseUrl, timeout=30)
-        }
+    }
+
 
 @app.post("/getCurrentCount")
 async def getCount(vectorstore: str):
@@ -156,11 +173,12 @@ async def getCount(vectorstore: str):
     df = pd.DataFrame(client.table("ConversAI_ChatbotInfo").select("*").execute().data)
     return {
         "currentCount": df[(df['username'] == username) & (df['chatbotname'] == chatbotName)]['charactercount'].iloc[0]
-        }
+    }
+
 
 @app.post("/getYoutubeTranscript")
 async def getYTTranscript(urls: str):
-    return getTranscript(urls = urls)
+    return getTranscript(urls=urls)
 
 
 @app.post("/analyzeData")
@@ -169,10 +187,10 @@ async def analyzeAndAnswer(query: str, file: UploadFile = File(...)):
     try:
         if extension in ["xls", "xlsx", "xlsm", "xlsb"]:
             df = pd.read_excel(io.BytesIO(await file.read()))
-            response = analyzeData(query = query, dataframe = df)
+            response = analyzeData(query=query, dataframe=df)
         elif extension == "csv":
             df = pd.read_csv(io.BytesIO(await file.read()))
-            response = analyzeData(query = query, dataframe = df)
+            response = analyzeData(query=query, dataframe=df)
         else:
             response = "INVALID FILE TYPE"
         return {
@@ -181,4 +199,4 @@ async def analyzeAndAnswer(query: str, file: UploadFile = File(...)):
     except:
         return {
             "output": "UNABLE TO ANSWER QUERY"
-        }
+        }

requirements.txt

@@ -24,4 +24,5 @@ pdf2image
 sentence-transformers
 supabase
 unstructured
-urllib3
+urllib3
+gtts

src/__init__.py

@@ -0,0 +1,19 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+
+import logging.config
+import yaml
+import os
+
+if os.path.exists("logs"):
+    pass
+else:
+    os.makedirs("logs")
+
+log_config_path = os.path.join(os.getcwd(), "logging_config.yaml")
+with open(log_config_path, 'r') as file:
+    config = yaml.safe_load(file.read())
+
+logging.config.dictConfig(config)

src/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/api/speech_api.py

@@ -0,0 +1,195 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import os
+import tempfile
+from fastapi.responses import JSONResponse
+from fastapi import Form
+from fastapi import UploadFile, HTTPException, status
+from src.models.models import TextToSpeechRequest
+from fastapi.routing import APIRouter
+from src.pipeline.speech_transcription_pipeline import SpeechTranscriptionPipeline
+from src import logging
+
+speech_translator_router = APIRouter(tags=["SpeechTranscription"])
+pipeline = SpeechTranscriptionPipeline()
+
+
+@speech_translator_router.post(
+    "/text_to_speech",
+    description="""
+    ** For language refer below points**
+    **Supported Locales:**
+
+    - **English:**
+      - **Australia:** 
+        - **Language:** en
+        - **TLD:** com.au
+      - **United Kingdom:**
+        - **Language:** en
+        - **TLD:** co.uk
+      - **United States:**
+        - **Language:** en
+        - **TLD:** us
+      - **Canada:**
+        - **Language:** en
+        - **TLD:** ca
+      - **India:**
+        - **Language:** en
+        - **TLD:** co.in
+      - **Ireland:**
+        - **Language:** en
+        - **TLD:** ie
+      - **South Africa:**
+        - **Language:** en
+        - **TLD:** co.za
+      - **Nigeria:**
+        - **Language:** en
+        - **TLD:** com.ng
+
+    - **French:**
+      - **Canada:** 
+        - **Language:** fr
+        - **TLD:** ca
+      - **France:**
+        - **Language:** fr
+        - **TLD:** fr
+
+    - **Mandarin:**
+      - **China Mainland:** 
+        - **Language:** zh-CN
+        - **TLD:** any
+      - **Taiwan:**
+        - **Language:** zh-TW
+        - **TLD:** any
+
+    - **Portuguese:**
+      - **Brazil:** 
+        - **Language:** pt
+        - **TLD:** com.br
+      - **Portugal:**
+        - **Language:** pt
+        - **TLD:** pt
+
+    - **Spanish:**
+      - **Mexico:** 
+        - **Language:** es
+        - **TLD:** com.mx
+      - **Spain:**
+        - **Language:** es
+        - **TLD:** es
+      - **United States:**
+        - **Language:** es
+        - **TLD:** us
+    """
+)
+async def text_to_speech(request: TextToSpeechRequest):
+    logging.info(f"Text to speech request received")
+    try:
+        audio_bytes = pipeline.text_to_speech(request.text, request.lang, request.tld)
+        if not audio_bytes:
+            logging.error(f"Audio generation failed.")
+            raise ValueError("Audio generation failed.")
+        logging.info(f"Text to speech request processed successfully")
+        return JSONResponse(content={"audio": audio_bytes, "status_code": status.HTTP_200_OK}, status_code=200)
+    except ValueError as ve:
+        logging.error(f"Error processing text to speech request: {str(ve)}")
+        raise HTTPException(status_code=400, detail=str(ve))
+    except Exception as e:
+        logging.error(f"Internal Server Error: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+
+
+@speech_translator_router.post(
+    "/speech_to_text",
+    description="""
+    ** Specify the language used in the audio **
+    **Supported Languages:**
+
+    **Major Languages:**
+    - **English:** en
+    - **Mandarin Chinese:** zh
+    - **Spanish:** es
+    - **French:** fr
+    - **German:** de
+    - **Italian:** it
+    - **Japanese:** ja
+    - **Korean:** ko
+    - **Russian:** ru
+    - **Portuguese:** pt
+    - **Arabic:** ar
+
+    **Additional Languages:**
+
+    - **Indic Languages:**
+      - **Hindi:** hi
+      - **Bengali:** bn
+      - **Tamil:** ta
+      - **Telugu:** te
+
+    - **Southeast Asian Languages:**
+      - **Vietnamese:** vi
+      - **Thai:** th
+      - **Indonesian:** id
+      - **Malay:** ms
+
+    - **African Languages:**
+      - **Swahili:** sw
+      - **Yoruba:** yo
+      - **Hausa:** ha
+
+    - **European Languages:**
+      - **Polish:** pl
+      - **Dutch:** nl
+      - **Swedish:** sv
+      - **Norwegian:** no
+    """
+)
+async def speech_to_text(audio: UploadFile, lang: str = Form(...)):
+    logging.info(f"Speech to text request received")
+    try:
+        audio_bytes = await audio.read()
+        if not audio_bytes:
+            logging.error(f"Empty audio file")
+            raise ValueError("Empty audio file")
+    except Exception as e:
+        logging.error(f"Invalid audio file {e}")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid audio file"
+        )
+
+    try:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
+            temp_audio_file.write(audio_bytes)
+            temp_audio_file_path = temp_audio_file.name
+            logging.info(f"Temporary audio file created at {temp_audio_file_path}")
+    except Exception as e:
+        logging.error(f"Could not process audio file{e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Could not process audio file"
+        )
+
+    try:
+        logging.info(f"Transcribing audio to text")
+        transcript = pipeline.speech_to_text(temp_audio_file_path, lang)
+    except FileNotFoundError as fnfe:
+        logging.error(f"Temporary file not found{fnfel}")
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Temporary file not found"
+        )
+    except Exception as e:
+        logging.error(f"Error processing speech-to-text: {str(e)}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Error processing speech-to-text"
+        )
+    finally:
+        logging.info(f"Cleaning up temporary audio file")
+        if os.path.exists(temp_audio_file_path):
+            os.remove(temp_audio_file_path)
+
+    return JSONResponse(content={"transcript": transcript, "status_code": status.HTTP_200_OK}, status_code=200)

src/components/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/components/speech_to_text.py

@@ -0,0 +1,48 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import torch
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+
+
+class SpeechToText:
+    def __init__(self):
+        self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        self.torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+
+        model_id = "openai/whisper-large-v3"
+
+        self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            model_id, torch_dtype=self.torch_dtype, low_cpu_mem_usage=True, use_safetensors=True
+        ).to(self.device)
+        self.processor = AutoProcessor.from_pretrained(model_id)
+        self.speech_to_text_pipeline = self.pipeline()
+
+    def pipeline(self):
+        pipe = pipeline(
+            "automatic-speech-recognition",
+            model=self.model,
+            tokenizer=self.processor.tokenizer,
+            feature_extractor=self.processor.feature_extractor,
+            max_new_tokens=128, # max number of tokens to generate at a time
+            chunk_length_s=30, # length of audio chunks to process at a time
+            batch_size=16, # number of chunks to process at a time
+            return_timestamps=True,
+            torch_dtype=self.torch_dtype,
+            device=self.device,
+
+        )
+        return pipe
+
+    def transcribe_audio(self, audio, language: str = "en"):
+        """
+        This function is for transcribing audio to text.
+        :param audio: upload your audio file
+        :param language:  choose the languaage of the audio file
+        :return:
+        """
+
+        result = self.speech_to_text_pipeline(audio, return_timestamps=True,
+                                              generate_kwargs={"language": language, "task": "translate"})
+        return result["chunks"], result["text"]

src/components/text_to_speech_gtts.py

@@ -0,0 +1,33 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+import base64
+from io import BytesIO
+
+from gtts import gTTS
+from gtts.tokenizer import pre_processors
+
+
+class TextToSpeech:
+    def __init__(self):
+        self.preprocessing = [pre_processors.tone_marks, pre_processors.end_of_line, pre_processors.word_sub,
+                              pre_processors.abbreviations]
+
+    def conversion(self, text: str, lang: str, tld: str) -> str:
+        """
+        Convert text to speech and return the Base64-encoded MP3 data.
+        :param text: The text to convert to speech.
+        :param lang: The language in which to convert the text.
+        :return: Base64-encoded MP3 data as a string.
+        """
+        tts = gTTS(text=text, lang=lang, slow=False, tld=tld, pre_processor_funcs=self.preprocessing)
+        mp3_fp = BytesIO()
+        tts.write_to_fp(mp3_fp)
+        mp3_fp.seek(0)
+
+        mp3_binary = mp3_fp.getvalue()
+
+        base64_mp3 = base64.b64encode(mp3_binary).decode("utf-8")
+
+        return base64_mp3

src/models/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/models/models.py

@@ -0,0 +1,16 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from fastapi import UploadFile
+from pydantic import BaseModel
+
+
+class TextToSpeechRequest(BaseModel):
+    text: str
+    lang: str
+    tld: str
+
+
+class SpeechToTextRequest(BaseModel):
+    lang: str

src/pipeline/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/pipeline/speech_transcription_pipeline.py

@@ -0,0 +1,32 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""
+from src.components.speech_to_text import SpeechToText
+from src.components.text_to_speech_gtts import TextToSpeech
+
+
+class SpeechTranscriptionPipeline:
+    def __init__(self):
+        self.speech_to_text_ = SpeechToText()
+        self.text_to_speech_ = TextToSpeech()
+
+    def text_to_speech(self, text: str, lang: str, tld: str) -> str:
+        """
+        Convert text to speech.
+        :param text: The text to convert to speech.
+        :param lang: The language in which to convert the text.
+        :return: The speech representation of the text.
+        """
+        speech = self.text_to_speech_.conversion(text, lang, tld)
+        return speech
+
+    def speech_to_text(self, audio, lang: str) -> str:
+        """
+        Convert speech to text.
+        :param audio: The audio data to convert to text.
+        :param lang: The language in which the audio is spoken.
+        :return: The text representation of the audio.
+        """
+        transcript_with_timestamp, transcript = self.speech_to_text_.transcribe_audio(audio=audio, language=lang)
+        return transcript

src/utils/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-07-31
+"""

src/utils/utils.py

@@ -0,0 +1,11 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-02
+"""
+import yaml
+
+
+def load_config(file_path):
+    with open(file_path, 'r') as file:
+        config = yaml.safe_load(file)
+    return config