Upload 9 files

ChatBotApp.py

@@ -0,0 +1,399 @@
+import streamlit as st
+from llm_chatbot import LLMChatBot
+from streamlit_option_menu import option_menu
+import speech_recognition as sr
+import pyttsx3
+import os
+import getpass
+from uuid import uuid4
+import faiss
+import numpy as np
+import requests
+import io
+import warnings
+import torch
+import pickle
+import asyncio
+import json
+from git import Repo
+from rich import print as rp
+from typing import Union, List, Generator, Any, Mapping, Optional, Dict
+from requests.sessions import RequestsCookieJar
+from dotenv import load_dotenv, find_dotenv
+from langchain import hub
+from langchain_core.documents import Document
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains import create_retrieval_chain
+from langchain_community.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter, Language
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma, FAISS
+from langchain.vectorstores.base import VectorStore
+from langchain.retrievers import MultiQueryRetriever
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.llms import BaseLLM
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.document_compressors import LLMChainExtractor
+from langchain.retrievers.document_compressors import DocumentCompressorPipeline
+from langchain_community.document_transformers import EmbeddingsRedundantFilter
+from langchain_text_splitters import CharacterTextSplitter
+from langchain.retrievers.document_compressors import EmbeddingsFilter
+from langchain.memory.buffer import ConversationBufferMemory
+from langchain.chains import StuffDocumentsChain, LLMChain, ConversationalRetrievalChain
+from uber_toolkit_class import UberToolkit
+from glob import glob
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+import plotly.express as px
+from plotly.subplots import make_subplots
+import plotly.io as pio
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import MinMaxScaler
+from langchain_core.documents import Document
+from scipy.stats import gaussian_kde
+from huggingface_hub import InferenceClient
+from hugchat import hugchat
+from hugchat.login import Login
+from hugchat.message import Message
+from hugchat.types.assistant import Assistant
+from hugchat.types.model import Model
+from hugchat.types.message import MessageNode, Conversation
+from langchain_community.document_loaders import TextLoader
+from TTS.api import TTS
+import time
+from playsound import playsound
+from system_prompts import __all__ as prompts
+
+from profiler import VoiceProfileManager, VoiceProfile
+
+# Load environment variables
+load_dotenv(find_dotenv())
+
+class ChatbotApp:
+    def __init__(self, email, password, default_llm=1):
+        self.email = email
+        self.password = password
+        self.default_llm = default_llm
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name="all-MiniLM-L6-v2",
+            model_kwargs={'device': 'cpu'},
+            encode_kwargs={'normalize_embeddings': True}
+        )
+        self.vectorstore = None
+       
+
+
+    def create_vectorstore_from_github(self):
+        repo_url = "YOUR_REPO_URL"
+        local_repo_path = self.clone_github_repo(repo_url)
+        loader = DirectoryLoader(path=local_repo_path, glob=f"**/*", show_progress=True, recursive=True)
+        loaded_files = loader.load()
+        documents = [Document(page_content=file_content) for file_content in loaded_files]
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+        split_documents = text_splitter.split_documents(documents)
+        texts = [doc.page_content for doc in split_documents]
+        print(f"Texts for embedding: {texts}")  # Debug print
+        self.vectorstore = FAISS.from_texts(texts, self.embeddings)
+    
+    def create_vectorstore(self, docs):
+        
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+        # Wrap text content in Document objects
+        documents = [Document(page_content=doc) for doc in docs]
+        # Split documents using the text splitter
+        split_documents = text_splitter.split_documents(documents)
+        # Convert split documents back to plain text
+        texts = [doc.page_content for doc in split_documents]
+        vectorstore = FAISS.from_texts(texts, self.setup_embeddings())
+        return vectorstore
+    
+    def setup_session_state(self):
+        if 'chat_history' not in st.session_state:
+            st.session_state.chat_history = []
+        if 'voice_mode' not in st.session_state:
+            st.session_state.voice_mode = False
+        if 'vectorstore' not in st.session_state:
+            st.session_state.vectorstore = None
+        if 'retriever' not in st.session_state:
+            st.session_state.retriever = None
+        if 'compression_retriever' not in st.session_state:
+            st.session_state.compression_retriever = None
+
+    def text_to_speech(self, text):
+        self.engine.say(text)
+        self.engine.runAndWait()
+
+    def speech_to_text(self):
+        r = sr.Recognizer()
+        with sr.Microphone() as source:
+            st.write("Listening...")
+            audio = r.listen(source)
+            try:
+                text = r.recognize_google(audio)
+                return text
+            except:
+                return "Sorry, I didn't catch that."
+
+
+
+    def setup_embeddings(self):
+        return HuggingFaceEmbeddings(
+            model_name="all-MiniLM-L6-v2",
+            model_kwargs={'device': 'cpu'},
+            encode_kwargs={'normalize_embeddings': True}
+        )
+ 
+    def create_vector_store(self, docs):
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+        # Wrap text content in Document objects
+        documents = [Document(page_content=doc) for doc in docs]
+        # Split documents using the text splitter
+        split_documents = text_splitter.split_documents(documents)
+        print(f"Split documents: {split_documents}")  # Debug print
+        # Convert split documents back to plain text
+        texts = [doc.page_content for doc in split_documents]
+        print(f"Texts: {texts}")  # Debug print
+        if not texts:
+            print("No valid texts found for embedding. Check your repository content.")
+            return
+
+        try:
+            self.vectorstore = FAISS.from_texts(texts, self.embeddings)
+            print("Vector store created successfully")
+        except Exception as e:
+            print(f"Error creating vector store: {str(e)}")
+
+
+    def setup_retriever(self, k=5, similarity_threshold=0.76):
+        self.retriever = st.session_state.vectorstore.as_retriever(k=k)
+        splitter = CharacterTextSplitter(chunk_size=300, chunk_overlap=0, separator=". ")
+        redundant_filter = EmbeddingsRedundantFilter(embeddings=self.setup_embeddings())
+        relevant_filter = EmbeddingsFilter(embeddings=self.setup_embeddings(), similarity_threshold=similarity_threshold)
+        pipeline_compressor = DocumentCompressorPipeline(
+            transformers=[splitter, redundant_filter, relevant_filter]
+        )
+        st.session_state.compression_retriever = ContextualCompressionRetriever(base_compressor=pipeline_compressor, base_retriever=self.retriever)
+
+    def create_retrieval_chain(self):
+        rag_prompt = hub.pull("langchain-ai/retrieval-qa-chat")
+        combine_docs_chain = create_stuff_documents_chain(self.llm, rag_prompt)
+        self.high_retrieval_chain = create_retrieval_chain(st.session_state.compression_retriever, combine_docs_chain)
+        self.low_retrieval_chain = create_retrieval_chain(self.retriever, combine_docs_chain)
+
+    def setup_tts(self, model_name="tts_models/en/ljspeech/fast_pitch"):
+        self.tts = TTS(model_name=model_name, progress_bar=False, vocoder_path='vocoder_models/en/ljspeech/univnet')
+
+    def setup_speech_recognition(self):
+        self.recognizer = sr.Recognizer()
+
+    def setup_folders(self):
+        self.dirs = ["test_input", "vectorstore", "test"]
+        for d in self.dirs:
+            os.makedirs(d, exist_ok=True)
+
+    def send_message(self, message, web=False):
+        message_result = self.llm.chat(message, web_search=web)
+        return message_result.wait_until_done()
+
+    def stream_response(self, message, web=False, stream=False):
+        responses = []
+        for resp in self.llm.query(message, stream=stream, web_search=web):
+            responses.append(resp['token'])
+        return ' '.join(responses)
+
+    def web_search(self, text):
+        result = self.send_message(text, web=True)
+        return result
+
+    def retrieve_context(self, query: str):
+        context = []
+        lowres = self.retriever._get_relevant_documents(query)
+        highres = st.session_state.compression_retriever.get_relevant_documents(query)
+        context = "\n".join([doc.page_content for doc in lowres + highres])
+        return context
+
+    def get_conversation_chain(self):
+        EMAIL = os.getenv("EMAIL")
+        PASSWD = os.getenv("PASSWD")
+        model = 1
+        self.llm = LLMChatBot(EMAIL, PASSWD, default_llm=model)
+        self.llm.create_new_conversation(system_prompt=self.llm.default_system_prompt, switch_to=True)
+
+        memory = ConversationBufferMemory(memory_key='chat_history', return_messages=True)
+        conversation_chain = ConversationalRetrievalChain.from_llm(
+            llm=self.llm,
+            retriever=st.session_state.vectorstore.as_retriever(),
+            memory=memory
+        )
+        return conversation_chain
+
+    async def handle_user_input(self, user_input):
+        response = st.session_state.conversation({'question': user_input})
+        st.session_state.chat_history = response['chat_history']
+
+        for i, message in enumerate(st.session_state.chat_history):
+            if i % 2 == 0:
+                st.write(f"Human: {message.content}")
+            else:
+                st.write(f"AI: {message.content}")
+                if st.session_state.voice_mode:
+                    self.text_to_speech(message.content)
+
+    def clone_github_repo(self, repo_url, local_path='./repo'):
+        if os.path.exists(local_path):
+            st.write("Repository already cloned.")
+            return local_path
+        Repo.clone_from(repo_url, local_path)
+        return local_path
+    
+    def glob_recursive_multiple_extensions(base_dir, extensions):
+        all_files = []
+        for ext in extensions:
+            pattern = os.path.join(base_dir, '**', f'*.{ext}')
+            files = glob(pattern, recursive=True)
+            all_files.extend(files)
+        return all_files
+
+    def load_documents_from_github(self, repo_url, file_types=['*.py', '*.md', '*.txt', '*.html']):
+        local_repo_path = self.clone_github_repo(repo_url)
+        globber=f"**/*/{{{','.join(file_types)}}}"
+        rp(globber)
+        loader = DirectoryLoader(path=local_repo_path, glob=globber, show_progress=True, recursive=True,loader_cls=TextLoader)
+        loaded_files = loader.load()
+        st.write(f"Nr. files loaded: {len(loaded_files)}")
+        print(f"Loaded files: {len(loaded_files)}")  # Debug print
+
+        # Convert the loaded files to Document objects
+        documents = [Document(page_content=file_content) for file_content in loaded_files]
+        print(f"Documents: {documents}")  # Debug print
+
+        return documents
+    
+    def split_documents(self, documents, chunk_s=512, chunk_o=0):
+        split_docs = []
+        splitter=None
+        for doc in documents:
+            ext = os.path.splitext(getattr(doc, 'source', '') or getattr(doc, 'filename', ''))[1].lower()
+            if ext == '.py':
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.PYTHON, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            elif ext in ['.md', '.markdown']:
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.MARKDOWN, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            elif ext in ['.html', '.htm']:
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.HTML, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            else:
+                splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_s, chunk_overlap=chunk_o)
+            split_docs.extend(splitter.split_documents([doc]))
+        return split_docs, splitter
+
+    def visualize_vectorstore(self):
+        if st.session_state.vectorstore is None:
+            st.write("Vectorstore is not initialized.")
+            return
+
+        documents = st.session_state.vectorstore.get_all_documents()
+        embeddings = [doc.embedding for doc in documents]
+
+        pca = PCA(n_components=3)
+        embeddings_3d = pca.fit_transform(embeddings)
+
+        scaler = MinMaxScaler()
+        embeddings_3d_normalized = scaler.fit_transform(embeddings_3d)
+
+        colors = embeddings_3d_normalized[:, 0]
+
+        hover_text = [f"Document {i}:<br>{doc.page_content[:100]}..." for i, doc in enumerate(documents)]
+
+        fig = go.Figure(data=[go.Scatter3d(
+            x=embeddings_3d_normalized[:, 0],
+            y=embeddings_3d_normalized[:, 1],
+            z=embeddings_3d_normalized[:, 2],
+            mode='markers',
+            marker=dict(
+                size=5,
+                color=colors,
+                colorscale='Viridis',
+                opacity=0.8
+            ),
+            text=hover_text,
+            hoverinfo='text'
+        )])
+
+        fig.update_layout(
+            title="Interactive 3D Vectorstore Document Distribution",
+            scene=dict(
+                xaxis_title="PCA Component 1",
+                yaxis_title="PCA Component 2",
+                zaxis_title="PCA Component 3"
+            ),
+            width=800,
+            height=600,
+        )
+
+        st.plotly_chart(fig)
+
+    def chatbot_page(self):
+        st.title("Chatbot")
+
+        # Toggle for voice mode
+        st.session_state.voice_mode = st.toggle("Voice Mode")
+
+        # File uploader for context injection
+        uploaded_file = st.file_uploader("Choose a file for context injection")
+        if uploaded_file is not None:
+            documents = [uploaded_file.read().decode()]
+            st.session_state.vectorstore = self.create_vector_store(documents)
+            st.session_state.conversation = self.get_conversation_chain()
+
+        # GitHub repository URL input
+        repo_url = st.text_input("Enter GitHub repository URL")
+        if repo_url:
+            documents = self.load_documents_from_github(repo_url)
+            split_docs, _ = self.split_documents(documents)
+            st.session_state.vectorstore = self.create_vector_store(split_docs)
+            st.session_state.conversation = self.get_conversation_chain()
+
+        # Chat interface
+        user_input = st.text_input("You: ", key="user_input")
+
+        if user_input:
+            asyncio.run(self.handle_user_input(user_input))
+
+        if st.session_state.voice_mode:
+            if st.button("Speak"):
+                user_speech = self.speech_to_text()
+                st.text_input("You: ", value=user_speech, key="user_speech_input")
+                if user_speech != "Sorry, I didn't catch that.":
+                    asyncio.run(self.handle_user_input(user_speech))
+
+    def dashboard_page(self):
+        st.title("Dashboard")
+
+        if st.session_state.vectorstore is not None:
+            st.write("Vectorstore Visualization")
+            self.visualize_vectorstore()
+        else:
+            st.write("Vectorstore is not initialized. Please add documents in the Chatbot page.")
+
+    def main(self):
+        st.set_page_config(page_title="Enhanced Multi-page Chatbot App", layout="wide")
+
+        # Sidebar navigation
+        with st.sidebar:
+            selected = option_menu(
+                menu_title="Navigation",
+                options=["Chatbot", "Dashboard"],
+                icons=["chat", "bar-chart"],
+                menu_icon="cast",
+                default_index=0,
+            )
+
+        if selected == "Chatbot":
+            self.chatbot_page()
+        elif selected == "Dashboard":
+            self.dashboard_page()
+
+
+if __name__ == "__main__":
+    app = ChatbotApp(os.getenv("EMAIL"),os.getenv("PASSWD"))
+    app.main()

FaissStorage.py

@@ -0,0 +1,954 @@
+import os
+import tempfile
+from datetime import datetime
+import webbrowser
+from tkinter import Toplevel
+import warnings
+import faiss,logging
+import numpy as np
+import wandb
+from typing import List, Dict, Any, Optional, Union
+from git import Repo
+import plotly.graph_objects as go
+import numpy as np
+from sklearn.decomposition import PCA
+import requests
+from rich import print as rp
+from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn
+from dotenv import load_dotenv, find_dotenv
+import speech_recognition
+from TTS.api import TTS
+from sklearn.decomposition import PCA
+from playsound import playsound
+from hugchat import hugchat
+from hugchat.login import Login
+from langchain_core.documents import Document
+
+
+from langchain_community.llms.huggingface_endpoint import HuggingFaceEndpoint
+from langchain_community.llms.huggingface_hub import HuggingFaceHub
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains import create_retrieval_chain
+from langchain_community.document_loaders import (
+    PyPDFLoader,
+    UnstructuredHTMLLoader,
+    UnstructuredWordDocumentLoader,
+    TextLoader,
+    PythonLoader
+)
+from langchain.retrievers import TimeWeightedVectorStoreRetriever
+from langchain_community.docstore.in_memory import InMemoryDocstore
+from langchain.text_splitter import RecursiveCharacterTextSplitter, Language,CharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.vectorstores.base import VectorStore
+from langchain.retrievers import MultiQueryRetriever, ContextualCompressionRetriever
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.retrievers.document_compressors import LLMChainExtractor, DocumentCompressorPipeline
+from langchain_community.document_transformers import EmbeddingsRedundantFilter
+from langchain.retrievers.document_compressors import EmbeddingsFilter
+import plotly.graph_objs as go
+
+
+from langchain.chains import LLMChain
+# Load environment variables
+load_dotenv(find_dotenv())
+warnings.filterwarnings("ignore")
+os.environ['FAISS_NO_AVX2'] = '1'
+os.environ["USER_AGENT"] = os.getenv("USER_AGENT")
+os.environ["HUGGINGFACEHUB_API_TOKEN"] = os.getenv("HUGGINGFACEHUB_API_TOKEN")
+wandb.require("core")
+# Import system prompts
+from system_prompts import __all__ as prompts
+
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, GPT2LMHeadModel, GPT2TokenizerFast
+from langchain_huggingface import HuggingFacePipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+
+
+
+class LLMChatBot:
+    def __init__(self, email, password, cookie_path_dir='./cookies/', default_llm=1, default_system_prompt='default_rag_prompt'):
+        self.email = email
+        self.password = password
+        self.current_model = 1
+        self.current_system_prompt=default_system_prompt
+        self.cookie_path_dir = cookie_path_dir
+        self.cookies = self.login()
+        self.default_llm = default_llm
+        self.chatbot = hugchat.ChatBot(cookies=self.cookies.get_dict(), default_llm=default_llm,system_prompt=prompts[default_system_prompt])
+        self.conversation_id=None
+        self.check_conv_id(self.conversation_id)
+        rp("[self.conversation_id:{self.conversation_id}]")
+
+
+    def check_conv_id(self, id=None):
+        if not self.conversation_id and not id:
+            self.conversation_id = self.chatbot.new_conversation(modelIndex=self.current_model,system_prompt=self.current_system_prompt)
+        else:
+            if id:
+                self.conversation_id=id
+                self.chatbot.change_conversation(self.conversation_id)
+            elif not self.chatbot.get_conversation_info(self.conversation_id) == self.chatbot.get_conversation_info():
+                self.chatbot.change_conversation(self.conversation_id)
+        
+        return self.conversation_id        
+
+    def login(self):
+        rp("Attempting to log in...")
+        sign = Login(self.email, self.password)
+        try:
+            cookies = sign.login(cookie_dir_path=self.cookie_path_dir, save_cookies=True)
+            rp("Login successful!")
+            return cookies
+        except Exception as e:
+            rp(f"Login failed: {e}")
+            rp("Attempting manual login with requests...")
+            self.manual_login()
+            raise
+
+    def manual_login(self):
+        login_url = "https://huggingface.co/login"
+        session = requests.Session()
+        response = session.get(login_url)
+        rp("Response Cookies:", response.cookies)
+        rp("Response Content:", response.content.decode())
+        
+        csrf_token = response.cookies.get('csrf_token')
+        if not csrf_token:
+            rp("CSRF token not found in cookies.")
+            return
+        
+        login_data = {
+            'email': self.email,
+            'password': self.password,
+            'csrf_token': csrf_token
+        }
+        
+        response = session.post(login_url, data=login_data)
+        if response.ok:        
+
+            rp("Manual login successful!")
+        else:
+            rp("Manual login failed!")
+
+    def setup_speech_recognition(self):
+        self.recognizer = speech_recognition.Recognizer()
+    
+    def setup_tts(self, model_name="tts_models/en/ljspeech/fast_pitch"):
+        self.tts = TTS(model_name=model_name)
+
+    def chat(self, message):
+        return self.chatbot.chat(message)
+    
+    def query(self,message, web_search=False, stream=False,use_cache=True):
+        return self.chatbot.query(
+            text=message,
+            web_search = web_search,
+            temperature = 0.1,
+            top_p = 0.95,
+            repetition_penalty = 1.2,
+            top_k = 50,
+            truncate = 1000,
+            watermark = False,
+            max_new_tokens = 1024,
+            stop = ["</s>"],
+            return_full_text = False,
+            stream = stream,
+            _stream_yield_all = False,
+            use_cache = False,
+            is_retry = False,
+            retry_count = 5,
+            conversation = None
+        )
+    
+    def stream_response(self, message):
+        for resp in self.query(message, stream=True):
+            rp(resp)
+
+    def web_search(self, query):
+        query_result = self.query(query, web_search=True)
+        results = []
+        for source in query_result.web_search_sources:
+            results.append({
+                'link': source.link,
+                'title': source.title,
+                'hostname': source.hostname
+            })
+        return results
+
+    def create_new_conversation(self, switch_to=True):
+        return self.chatbot.new_conversation(switch_to=switch_to, modelIndex=self.current_model, system_prompt=self.current_system_prompt)
+
+    def get_remote_conversations(self):
+        return self.chatbot.get_remote_conversations(replace_conversation_list=True)
+
+    def get_local_conversations(self):
+        return self.chatbot.get_conversation_list()
+
+    def get_available_models(self):
+        return self.chatbot.get_available_llm_models()
+
+    def switch_model(self, index):
+        self.chatbot.switch_llm(index)
+
+    def switch_conversation(self, id):
+        self.conv_id = id
+        self.chatbot.change_conversation(self.conv_id)
+
+    def get_assistants(self):
+        return self.chatbot.get_assistant_list_by_page(1)
+
+    def switch_role(self, system_prompt, model_id=1):
+        self.chatbot.delete_all_conversations()
+        self.check_conv_id = self.chatbot.new_conversation(switch_to=True, system_prompt=system_prompt,  modelIndex=model_id)
+        return self.check_conv_id
+    
+    def __run__(self, message):
+        if not self.conversation_id:
+            self.conversation_id = self.chatbot.new_conversation(modelIndex=self.current_model,
+                                                                 system_prompt=self.current_system_prompt,
+                                                                 switch_to=True)
+        return self.query(message)
+    
+    def __call__(self, message):
+        if not self.conversation_id:
+            self.conversation_id = self.chatbot.new_conversation(modelIndex=self.current_model,
+                                                                 system_prompt=self.current_system_prompt,
+                                                                 switch_to=True)
+        return self.chat(message)
+
+
+
+
+class AdvancedVectorStore:
+    def __init__(self, 
+                 embedding_model: str = "all-MiniLM-L6-v2", 
+                 email: str = None, 
+                 password: str = None,
+                 chunk_size=384,
+                 chunk_overlap=0, 
+                 device='cpu',
+                 normalize_embeddings=True,
+                 log_level=logging.INFO,
+                 log_file='AdvancedVectorStore.log',
+                 logs_dir='./logs',
+                 test_input='./test_input',
+                 test_output='./test_output',
+                 storage_dir='./vectorstore',
+                 knowledge_dir='./knowledge',
+                 repos_dir='./repos'
+    ):
+
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.device = device
+        self.basic_splitter= RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)
+        self.storage_dir=storage_dir
+        self.test_input=test_input
+        self.test_output=test_output
+        self.repos_dir=repos_dir
+        self.knowledge_dir=knowledge_dir
+        self.logs_dir=logs_dir
+        self.log_file=log_file
+        self.doc_ids = []
+        self.documents: List[Document] = []
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name=embedding_model,
+            model_kwargs={'device': self.device},
+            encode_kwargs={'normalize_embeddings': normalize_embeddings}
+        )
+
+        self.qwen_llm = HuggingFaceHub(repo_id="Qwen/Qwen2-0.5B-Instruct", model_kwargs={"temperature": 0.5, "max_length": 512})
+        self.llm = HuggingFaceHub(repo_id="google-t5/t5-small", model_kwargs={"temperature": 0.5, "max_length": 512})
+        self.alpaca_llm = HuggingFaceHub(repo_id="reasonwang/google-flan-t5-small-alpaca", model_kwargs={"temperature": 0.1, "max_length": 512})
+        self.chatbot_llm = LLMChatBot(email, password, default_system_prompt= 'copilot_prompt') if email and password else None
+
+        rp("create_indexed_vectorstore:")
+        print(self.alpaca_llm("What is Deep Learning?"))  
+
+        self.vectorstore, self.docstore, self.index = self.create_indexed_vectorstore(self.chunk_size)
+
+        self.document_count = 0
+        self.chunk_count = 0
+        self.setup_folders()     
+        self.setup_logging(log_level,os.path.join(self.logs_dir,self.log_file))
+        self.logger.info("Initializing AdvancedVectorStore")
+        self.set_bot_role()
+    
+    def setup_logging(self,level,file):
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(level)
+        # Create console handler and set level
+        ch = logging.StreamHandler()
+        ch.setLevel(level)
+        # Create file handler and set level
+        fh = logging.FileHandler(file)
+        fh.setLevel(level)
+        # Create formatter
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        # Add formatter to console handler
+        ch.setFormatter(formatter)
+        # Add formatter to file handler
+        fh.setFormatter(formatter)
+        # Add handlers to logger
+        self.logger.addHandler(ch)
+        self.logger.addHandler(fh)
+        self.logger.info("Done settingload_documents_folder up logger for {__name__} [AdvancedVectorStore]")
+        
+    def setup_folders(self):
+        self.dirs = [
+            self.test_input,
+            self.test_output,
+            self.logs_dir,
+            self.storage_dir,
+            self.knowledge_dir,
+            self.repos_dir
+        ]
+        for d in self.dirs:
+            os.makedirs(d, exist_ok=True)
+
+    def set_bot_role(self,prompt='default_rag_prompt',context="",history=""):
+        self.chatbot_llm.current_system_prompt = prompts[prompt].replace("<<VSCONTEXT>>",context).replace("<<WSCONTEXT>>",history)
+        self.current_conversation_id=self.chatbot_llm.chatbot.new_conversation(system_prompt=self.chatbot_llm.current_system_prompt,
+                                                                               modelIndex=self.chatbot_llm.current_model,
+                                                                               switch_to=True)
+        #self.logger.info(f"Setting Bot Role!\n[{prompt}]")
+        """ result=self.chatbot_llm("Confirm you understand the TASK.")
+        self.logger.info(f"Test results chatbot role set:{result}") """
+        #rp(f"[Result:{result}]")
+        
+
+    def load_documents(self, directory: str) -> None:
+        """Load documents from a directory with specific loaders for each file type."""
+        loaders = {
+            ".py": (PythonLoader, {}),
+            ".txt": (TextLoader, {}),
+            ".pdf": (PyPDFLoader, {}),
+            ".html": (UnstructuredHTMLLoader, {}),
+            ".docx": (UnstructuredWordDocumentLoader, {})
+        }
+
+        for root, _, files in os.walk(directory):
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_extension = os.path.splitext(file)[1].lower()
+
+                if file_extension in loaders:
+                    # Check if the file can be read as UTF-8
+                    try:
+                        with open(file_path, 'r', encoding='utf-8') as f:
+                            f.read()
+                    except (UnicodeDecodeError, IOError):
+                        rp(f"Skipping non-UTF-8 or unreadable file: {file_path}")
+                        continue
+
+                    loader_class, loader_args = loaders[file_extension]
+                    loader = loader_class(file_path, **loader_args)
+                    self.documents.extend(loader.load())
+
+    def split_documents(self) -> None:
+        """Split documents using appropriate splitters for each file type."""
+        splitters = {
+            ".py": RecursiveCharacterTextSplitter.from_language(language=Language.PYTHON, chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap),
+            ".txt": RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap),
+            ".pdf": RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap),
+            ".html": RecursiveCharacterTextSplitter.from_language(language=Language.HTML, chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap),
+            ".docx": RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap)
+        }
+
+        split_docs = []
+        for doc in self.documents:
+            file_extension = os.path.splitext(doc.metadata.get("source", ""))[1].lower()
+            splitter = splitters.get(file_extension, RecursiveCharacterTextSplitter(chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap))
+            split_docs.extend(splitter.split_documents([doc]))
+
+        self.documents = split_docs
+
+    def create_vectorstore(self, store_type: str = "FAISS") -> None:
+        """Create a vectorstore of the specified type."""
+        if store_type == "FAISS":
+            self.vectorstore = FAISS.from_documents(self.documents, self.embeddings)
+        else:
+            raise ValueError(f"Unsupported vectorstore type: {store_type}")
+    
+    def create_indexed_vectorstore(self,embedding_size):
+        rp("Creating indexed vectorstore...")
+        #embedding_size = 384  # Size for all-MiniLM-L6-v2 embeddings
+        index = faiss.IndexFlatL2(embedding_size)
+        docstore = InMemoryDocstore({})
+        vectorstore = FAISS(
+            self.embeddings.embed_query, 
+            index, 
+            docstore, 
+            {}
+        )
+        rp("Indexed vectorstore created.")
+        return vectorstore, docstore, index
+           
+    def get_self_query_retriever(self, k: int = 4) -> SelfQueryRetriever:
+        """Get a SelfQueryRetriever."""
+        if not self.vectorstore:
+            raise ValueError("Vectorstore not initialized. Call create_vectorstore() first.")
+        return SelfQueryRetriever.from_llm(
+            self.chatbot_llm.chatbot,
+            self.vectorstore,
+            document_contents="Document about various topics.",
+            metadata_field_info=[],
+            search_kwargs={"k": k}
+        )
+
+    
+    def get_contextual_t5_compression_retriever(self, k: int = 4, similarity_threshold=0.78) -> ContextualCompressionRetriever:
+        """Get a ContextualCompressionRetriever."""
+        base_compressor = LLMChainExtractor.from_llm(self.llm)
+        redundant_filter = EmbeddingsRedundantFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        relevant_filter = EmbeddingsFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        return ContextualCompressionRetriever(
+            name="CompressedRetriever",
+            base_compressor=DocumentCompressorPipeline(transformers=[self.basic_splitter, base_compressor, redundant_filter, relevant_filter]),
+            base_retriever=self.get_basic_retriever(k=k)
+        )
+    
+    def get_contextual_qwen_compression_retriever(self, k=4, similarity_threshold=0.78):
+        # Initialize the components for the compressor pipeline
+        base_compressor = LLMChainExtractor.from_llm(self.qwen_llm)
+        redundant_filter = EmbeddingsRedundantFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        relevant_filter = EmbeddingsFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        # Create the ContextualCompressionRetriever
+        return ContextualCompressionRetriever(
+            name="CompressedRetriever",
+            base_compressor= DocumentCompressorPipeline(transformers=[self.basic_splitter, base_compressor, redundant_filter, relevant_filter]),
+            base_retriever=self.get_basic_retriever(k=k)
+        )
+    
+    def get_contextual_compression_retriever(self, k: int = 4,similarity_threshold=0.78) -> ContextualCompressionRetriever:
+        """Get a ContextualCompressionRetriever."""
+        base_compressor = LLMChainExtractor.from_llm(self.alpaca_llm)
+        redundant_filter = EmbeddingsRedundantFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        relevant_filter = EmbeddingsFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        return ContextualCompressionRetriever(
+            name="CompressedRetriever",
+            base_compressor=DocumentCompressorPipeline(transformers=[self.basic_splitter, base_compressor, redundant_filter, relevant_filter]),
+            base_retriever=self.get_basic_retriever(k=k)
+        )
+    
+    
+    def get_basic_retriever(self, k: int = 4) -> VectorStore:
+        """Get a basic retriever from the vectorstore."""
+        if not self.vectorstore:
+            raise ValueError("Vectorstore not initialized. Call create_vectorstore() first.")
+        return self.vectorstore.as_retriever(search_kwargs={"k": k})
+    def get_multi_query_retriever(self, k: int = 4) -> MultiQueryRetriever:
+        """Get a MultiQueryRetriever."""
+        if not self.vectorstore:
+            raise ValueError("Vectorstore not initialized. Call create_vectorstore() first.")
+        return MultiQueryRetriever.from_llm(
+            retriever=self.vectorstore.as_retriever(search_kwargs={"k": k}),
+            llm=self.chatbot_llm
+        )
+    def get_timed_retriever(self, k=1, decay_rate=0.0000000000000000000000001):
+        return TimeWeightedVectorStoreRetriever(
+            vectorstore=self.vectorstore, decay_rate=decay_rate, k=k
+        )
+    
+    def set_current_retriever(self,mode='basic',k=4,sim_rate=0.78):
+        if mode == 'compressed':
+            retriever = self.get_contextual_compression_retriever(k, sim_rate)
+        elif mode == 'qwen_compressed':
+            retriever = self.get_contextual_qwen_compression_retriever(k, sim_rate)
+        elif mode == 't5_compressed':
+            retriever = self.get_contextual_t5_compression_retriever(k, sim_rate)
+        elif mode == 'self_query':
+            retriever = self.get_self_query_retriever(k)
+        elif mode == 'multi_query':
+            retriever = self.get_multi_query_retriever(k)
+        elif mode == 'time':
+            retriever = self.get_timed_retriever(k=1)
+        else:
+            retriever = self.get_basic_retriever(k)
+
+        #rp(retriever.get_prompts)
+        return retriever
+
+    def search(self, query: str, mode='basic', retriever: Optional[Any] = None, k: int = 4, sim_rate: float = 0.78) -> List[Document]:
+        """Search the vectorstore using the specified retriever."""
+        if not retriever:
+            retriever = self.set_current_retriever(mode=mode, k=k, sim_rate=sim_rate)
+        return retriever.get_relevant_documents(query)
+
+    def add_documents(self, documents: List[Document]) -> None:
+        import uuid
+        
+        """Add new documents to the existing vectorstore."""
+        with Progress(
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TextColumn("[green][progress.percentage]{task.percentage:>3.0f}%"),
+            TimeRemainingColumn()
+        ) as progress:
+            task = progress.add_task("[cyan]Adding documents to vectorstore...", total=len(documents))
+            
+            for id, doc in enumerate(documents):
+                #self.vectorstore.add_documents([doc])
+                metadata = doc.metadata
+                if not metadata:
+                    metadata = {}
+
+                metadata["last_accessed_at"] = datetime.now()
+                new_doc = Document(page_content=doc.page_content, metadata=metadata)
+                nr=id
+                id = str(uuid.uuid4())
+ 
+                
+                self.vectorstore.docstore.add({id: new_doc})
+                
+                self.doc_ids.append(id)
+                self.set_current_retriever(mode='time', k=1).add_documents([new_doc])
+                total = self.index.ntotal
+                #self.logger.info(f"Added doc to vectorstore {new_doc.metadata['last_accessed_at']} with {total} id's so far.")
+                self.logger.info(f"Added doc to docstore[{nr}/{len(self.documents)}] with Id:{id} Path:{new_doc.metadata['source']}")
+                progress.update(task, advance=1)
+
+        rp(f"Added {len(documents)} documents to the vectorstore with index in doc_ids.")
+
+    def delete_documents(self, document_ids: List[str]) -> None:
+        """Delete documents from the vectorstore by their IDs."""
+
+        for id in document_ids:
+            #self.logger.info(f"[Deleting DocumenId{id}...]")
+            self.vectorstore.delete(document_ids)
+            #self.logger.info(f"[Done! Saving Faiss...{id}]")
+
+    def save_vectorstore(self, path: str) -> None:
+        """Save the vectorstore to disk."""
+        if not self.vectorstore:
+            raise ValueError("Vectorstore not initialized. Call create_vectorstore() first.")
+        #self.logger.info("[Saving Faiss...]")
+        self.vectorstore.save_local(path)
+        #self.logger.info(f"[Done! Saving Faiss to:{path}]")
+
+    def load_vectorstore(self, path: str) -> None:
+        """Load the vectorstore from disk."""
+        #self.logger.info("Loading Faiss...")
+        self.vectorstore = FAISS.load_local(folder_path=path, 
+                                            embeddings=self.embeddings,
+                                            allow_dangerous_deserialization=True)
+        #self.logger.info(f"[Done! Loading Faiss from:{path}]")
+
+    def create_retrieval_chain(self, prompt: str = "default_rag_prompt", retriever: Optional[Any] = None) -> Any:
+        """Create a retrieval chain using the specified prompt and retriever."""
+        if not retriever:
+            retriever = self.get_basic_retriever()
+        
+        combine_docs_chain = create_stuff_documents_chain(self.chatbot_llm, prompt=prompts[prompt])
+        return create_retrieval_chain(retriever, combine_docs_chain)
+
+    def run_retrieval_chain(self, chain: Any, query: str) -> Dict[str, Any]:
+        """Run a retrieval chain with the given query."""
+        return chain.invoke({"input": query})
+
+    def generate_3d_scatterplot(self, num_points=1000):
+        """
+        Generate a 3D scatter plot of the vector store content and log it to wandb.
+        
+        :param num_points: Maximum number of points to plot (default: 1000)
+        :return: None (logs the plot to wandb)
+        """
+        all_docs = self.get_all_documents()
+
+        if not all_docs:
+            raise ValueError("No documents found in the vector store.")
+
+        # Extract vectors and metadata from documents
+        vectors = []
+        doc_ids = []
+        for doc in all_docs:
+            if hasattr(doc, 'embedding') and doc.embedding is not None:
+                vectors.append(doc.embedding)
+            else:
+                vectors.append(self.embeddings.embed_query(doc.page_content))
+            doc_ids.append(doc.metadata.get('id', 'Unknown'))
+
+        vectors = np.array(vectors)
+
+        # If we have more vectors than requested points, sample randomly
+        if len(vectors) > num_points:
+            indices = np.random.choice(len(vectors), num_points, replace=False)
+            vectors = vectors[indices]
+            doc_ids = [doc_ids[i] for i in indices]
+
+        # Perform PCA to reduce to 3 dimensions
+        pca = PCA(n_components=3)
+        vectors_3d = pca.fit_transform(vectors)
+
+
+        # Initialize wandb run
+        wandb.init(project="vector_store_visualization")
+
+        # Create the Plotly figure
+        fig = go.Figure(data=[go.Scatter3d(
+            x=vectors_3d[:, 0],
+            y=vectors_3d[:, 1],
+            z=vectors_3d[:, 2],
+            mode="markers",
+            marker=dict(
+                size=[28.666666666666668, 20.666666666666668, 15.333333333333334, 17.666666666666668, 19.0, 17.666666666666668, 26.0, 21.0, 21.666666666666668, 27.0, 21.666666666666668, 16.666666666666668, 27.0, 14.0, 29.666666666666668, 22.0, 16.0, 28.0, 27.0, 25.333333333333332],
+                color=[28.666666666666668, 20.666666666666668, 15.333333333333334, 17.666666666666668, 19.0, 17.666666666666668, 26.0, 21.0, 21.666666666666668, 27.0, 21.666666666666668, 16.666666666666668, 27.0, 14.0, 29.666666666666668, 22.0, 16.0, 28.0, 27.0, 25.333333333333332],
+                colorscale='Viridis',
+                showscale=True,
+                colorbar=dict(x=0),
+                cmin=14,
+                cmax=20
+            ),
+            text=self.doc_ids,
+            hoverinfo="text",
+            name="Document Vectors",
+        )])
+
+        # Update layout
+        fig.update_layout(
+            showlegend=True,
+            scene=dict(
+                xaxis_title="X Axis",
+                yaxis_title="Y Axis",
+                zaxis_title="Z Axis"
+            ),
+            width=1200,
+            height=1000,
+        )
+
+        # Log the plot to wandb
+        wandb.log({"3D Scatter Plot": fig})
+
+        # Finish the wandb run
+        wandb.finish()
+        
+    def load_documents_folder(self, folder_path):
+            rp("[Loading documents from cloned repository]")
+            self.load_documents(folder_path)
+            self.document_count=len(self.documents)
+            rp(f"Splitting {self.document_count} documents")
+            self.split_documents()
+            self.chunk_count=len(self.documents)
+            rp(f"Adding {self.chunk_count} document chunks to vectorstore")
+            self.add_documents(self.documents)
+
+    def load_github_repo(self, repo_url: str) -> None:
+        """
+        Clone a GitHub repository to a temporary folder, load documents, and remove the folder.
+        """
+        split=repo_url.split('/')
+        repo_name = split.pop()
+        author_name = split.pop()
+        new_repo_path=os.path.join(self.repos_dir,f"{author_name}_{repo_name}")
+
+        if not os.path.exists(new_repo_path):
+            rp(f'Cloning repository {repo_url} to {new_repo_path}')
+            Repo.clone_from(repo_url, new_repo_path)
+            
+            rp("Loading documents from cloned repository")
+            self.load_documents(new_repo_path)
+            
+            rp(f"Splitting {len(self.documents)} documents into chunks")
+            self.split_documents()
+            
+            rp(f"Adding {len(self.documents)} documents to vectorstore")
+            self.add_documents(self.documents)
+            self.save_vectorstore(self.storage_dir)
+            self.load_vectorstore(self.storage_dir)
+            rp("Temporary folder removed")
+        else:
+            rp(f"Repository {repo_url} already exists in {new_repo_path}")
+            self.load_vectorstore(self.storage_dir)
+
+
+    def get_all_documents(self):
+        """
+        Fetch all documents from the document store.
+        """
+        all_docs = []
+        
+        # Number of vectors in the index
+        num_vectors = self.index.ntotal
+
+        # Assuming 'd' is the dimensionality of the vectors
+        d = self.index.d
+        #rp(f"D:{d}")
+        # Retrieve all vectors (this part is straightforward if you have access to the original vectors)
+        retrieved_vectors = np.empty((num_vectors, d), dtype='float32')
+        for i in range(num_vectors):
+            retrieved_vectors[i] = self.index.reconstruct(i)
+
+        # Assuming you have a way to get the document IDs
+        # In a real scenario, you would maintain a mapping of FAISS index positions to document IDs
+        # Example: you might have an attribute like 'self.doc_ids' which is a list of IDs
+        retrieved_ids = self.doc_ids[:num_vectors]  # Ensure you have this attribute properly maintained
+        #rp(f"Retrieved ids{retrieved_ids}")
+        # Fetch documents using the retrieved IDs
+        retrieved_docs = [self.docstore.search(doc_id) for doc_id in retrieved_ids]
+
+        # Collect all documents
+        all_docs.extend(retrieved_docs)
+
+        #for doc_id, doc in zip(retrieved_ids, retrieved_docs):
+            #rp(f"ID: {doc_id}, Document.page_content: {doc.page_content}, Document.metadata: {doc.metadata}")
+
+        return all_docs
+
+
+    def test_chat(self,text,context='This is a chat with a nice Senior programmer.',history='Your Birth as fresh outof the box agent.'):
+        
+        self.set_bot_role(context=context,history=history)
+        
+        return self.chatbot_llm(text)
+    def chat(self, message: str) -> str:
+        """
+        Send a message to the HugChat bot and get a response.
+        
+        :param message: The message to send to the bot
+        :return: The bot's response
+        """
+        if not self.chatbot_llm:
+            raise ValueError("HugChat bot not initialized. Provide email and password when creating AdvancedVectorStore.")
+        return self.chatbot_llm.chat(message)
+
+    def setup_speech_recognition(self):
+        """Set up speech recognition for the HugChat bot."""
+        if not self.chatbot_llm:
+            raise ValueError("HugChat bot not initialized. Provide email and password when creating AdvancedVectorStore.")
+        self.chatbot_llm.setup_speech_recognition()
+
+    def setup_tts(self, model_name="tts_models/en/ljspeech/fast_pitch"):
+        """Set up text-to-speech for the HugChat bot."""
+        if not self.chatbot_llm:
+            raise ValueError("HugChat bot not initialized. Provide email and password when creating AdvancedVectorStore.")
+        self.chatbot_llm.setup_tts(model_name)
+
+    def voice_chat(self):
+        """
+        Initiate a voice chat session with the HugChat bot.
+        """
+        if not self.chatbot_llm or not hasattr(self.chatbot_llm, 'recognizer') or not hasattr(self.chatbot_llm, 'tts'):
+            raise ValueError("Speech recognition and TTS not set up. Call setup_speech_recognition() and setup_tts() first.")
+
+        rp("Voice chat initiated. Speak your message (or say 'exit' to end the chat).")
+        
+        while True:
+            with speech_recognition.Microphone() as source:
+                rp("Listening...")
+                audio = self.chatbot_llm.recognizer.listen(source)
+
+            try:
+                user_input = self.chatbot_llm.recognizer.recognize_google(audio)
+                rp(f"You said: {user_input}")
+
+                if user_input.lower() == 'exit':
+                    rp("Ending voice chat.")
+                    break
+
+                response = self.chat(user_input)
+                rp(f"Bot: {response}")
+
+                # Generate speech from the bot's response
+                speech_file = "bot_response.wav"
+                self.chatbot_llm.tts.tts_to_file(text=response, file_path=speech_file)
+                playsound(speech_file)
+                os.remove(speech_file)  # Clean up the temporary audio file
+
+            except speech_recognition.UnknownValueError:
+                rp("Sorry, I couldn't understand that. Please try again.")
+            except speech_recognition.RequestError as e:
+                rp(f"Could not request results from the speech recognition service; {e}")
+
+    def rag_chat(self, query: str, prompt: str = "default_rag_prompt") -> str:
+        """
+        Perform a RAG (Retrieval-Augmented Generation) chat using the vectorstore and HugChat bot.
+        
+        :param query: The user's query
+        :param prompt: The prompt to use for the retrieval chain (default: "default_rag_prompt")
+        :return: The bot's response
+        """
+        if not self.vectorstore:
+            raise ValueError("Vectorstore not initialized. Call create_vectorstore() first.")
+
+        retriever = self.get_basic_retriever()
+        chain = self.create_retrieval_chain(prompt, retriever)
+        result = self.run_retrieval_chain(chain, query)
+        return result['answer']
+    
+    def search_web(self):
+        search_query = input("Enter your web search query: ")
+        future_date = "July 12, 2024"
+        search_url = f"https://www.google.com/search?q={search_query}+before:{future_date}"
+        webbrowser.open(search_url)
+        rp(f"Search results for '{search_query}' on {future_date}:")
+        rp("=" * 50)
+        rp(search_url)
+        rp("=" * 50)
+
+    def advanced_rag_chatbot(self):
+        rp("Welcome to the Advanced RAG Chatbot!")
+        rp("This chatbot uses a compressed retriever and integrates all components of the vector store.")
+        rp("Type 'exit' to end the conversation.")
+
+        # Ensure the vectorstore is initialized
+        if self.vectorstore is None:
+            rp("Initializing vector store...")
+            self.vectorstore, self.docstore, self.index = self.create_indexed_vectorstore(self.chunk_size)
+
+        # Create a compressed retriever
+        # compressed_retriever = self.get_contextual_compression_retriever(k=5, similarity_threshold=0.75)
+        mode='basic'
+        k=5
+        similarity_threshold=0.75
+        retriever = self.set_current_retriever(mode=mode, k=k, sim_rate=similarity_threshold)
+        #basic_retriever = self.get_basic_retriever(k=4)
+        # Initialize conversation history
+        conversation_history = []
+
+        while True:
+            user_input = input("\nYou: ").strip()
+            if user_input.lower() == 'exit':
+                rp("Thank you for using the Advanced RAG Chatbot. Goodbye!")
+                break
+
+            rp("# Step 1: Retrieve relevant documents")
+            retrieved_docs = self.get_basic_retriever(k=4).get_relevant_documents(user_input)
+
+            rp("# Step 2: Prepare context from retrieved documents")
+            context = "\n".join([doc.page_content for doc in retrieved_docs])
+
+            rp("# Step 3: Prepare the prompt")
+            #prompt = prompts['default_rag_prompt']
+            self.set_bot_role(context=context, history=' '.join(conversation_history[-5:]))
+            rp("# Step 4: Generate response using the chatbot")
+            response = self.chatbot_llm(f"User:{user_input}\n")
+
+            rp(f"Chatbot: {response}")
+
+            # Update conversation history
+            conversation_history.append(f"User: {user_input}")
+            conversation_history.append(f"Chatbot: {response}")
+
+            # Step 5: Demonstrate use of individual components
+            rp("\nAdditional Information:")
+            rp(f'- Number of documents in docstore: {len(self.docstore.search("* *"))}')
+            rp(f"- Number of vectors in index: {self.index.ntotal}")
+            
+            # Demonstrate direct use of vectorstore for similarity search
+            similar_docs = self.vectorstore.similarity_search(user_input, k=1)
+            similar_docs = self.vectorstore.similarity_search_with_relevance_scores(user_input,k=1)
+            if similar_docs:
+                rp(type(similar_docs))
+                rp(f"-[Most similar document: [{similar_docs[0].metadata.get('source', 'Unknown')}]]-")
+            
+            # Generate a 3D scatter plot of the vectorstore content
+            #avs.generate_3d_scatterplot_wandb()
+            avs.generate_3d_scatterplot()
+            
+            # Optional: Add user feedback loop
+            feedback = input("Was this response helpful? (yes/no): ").strip().lower()
+            if feedback == 'no':
+                rp("I'm sorry the response wasn't helpful. Let me try to improve it.")
+                # Here you could implement logic to refine the response or adjust the retrieval process
+                with open(file="./feedback_NO.txt",mode="a+")as f:
+                    f.write(f"chat_feedback_NO\nChatHistory--->{' '.join(conversation_history[-10:])}")
+
+
+
+# Example usage:
+if __name__ == "__main__":
+    email = os.getenv("EMAIL")
+    password = os.getenv("PASSWD")
+    github_token = os.getenv("GITHUB_TOKEN")
+    
+    # Initialize AdvancedVectorStore with HugChat bot
+    avs = AdvancedVectorStore(email=email, password=password)
+
+    # Create the indexed vectorstore
+    #avs.create_indexed_vectorstore()
+
+    # Clone a GitHub repository and load its contents
+    
+    # avs.load_documents_folder("/nr_ywo/coding/voice_chat_rag_web/venv/lib/python3.10/site-packages/huggingface_hub/inference")  
+    avs.load_documents_folder("/nr_ywo/coding/voice_chat_rag_web/venv/lib/python3.10/site-packages/hugchat")  
+    avs.load_documents_folder("/nr_ywo/coding/voice_chat_rag_web/venv/lib/python3.10/site-packages/langchain/agents")  
+    avs.load_documents_folder("/nr_ywo/coding/voice_chat_rag_web/venv/lib/python3.10/site-packages/langchain_experimental/autonomous_agents")  
+ 
+    #avs.chatbot_llm.load_documents("/nr_ywo/coding/voice_chat_rag_web/test_input")
+    # avs.load_github_repo("https://github.com/bxck75/voice_chat_rag_web")
+    avs.save_vectorstore(path=avs.storage_dir)
+    avs.load_vectorstore(path=avs.storage_dir)
+    # rp document and chunk counts
+    #rp(f"Total documents: {avs.chunk_count / avs.chunk_size}")
+    #rp(f"Total chunks: {avs.chunk_count}")
+    #avs.logger.info(avs.chatbot_llm.current_model)
+    #avs.logger.info(avs.chatbot_llm.current_system_prompt)
+
+    retriever=avs.set_current_retriever(mode='basic',k=4)
+    comptriever=avs.set_current_retriever(mode='compression',k=4,sim_rate=0.87)
+    timetriever=avs.set_current_retriever(mode='time',k=1)
+
+    q="Demonstrate your knowledge of developing advanced AI scripts in OOP python. try to come up with cutting edge ideas"
+    rel_docs=retriever.invoke(input=q)
+    #okrp(f"[Raw Knowledge Retrieved:{rel_docs}]")
+      # Start the advanced RAG chatbot
+    avs.advanced_rag_chatbot()
+
+    # Perform a RAG chat
+    #rag_response = avs.rag_chat(query="Explain the concept of neural networks.")
+    #rp("RAG chat response:", rag_response)
+
+    # Set up speech recognition and TTS for voice chat
+    #avs.setup_speech_recognition()
+    #avs.setup_tts()
+
+    # Start a voice chat session
+    #avs.voice_chat()
+    """ 
+        # Using different retrievers
+        multi_query_retriever = avs.get_multi_query_retriever()
+        results = avs.search("What is deep learning?", mode="multi_query")
+        rp("Multi-query retriever results:", results)
+
+        self_query_retriever = avs.get_self_query_retriever()
+        results = avs.search("Find documents about reinforcement learning", self_query_retriever)
+        rp("Self-query retriever results:", results)
+
+        contextual_compression_retriever = avs.get_contextual_compression_retriever()
+        results = avs.search("Explain the difference between supervised and unsupervised learning", contextual_compression_retriever)
+        rp("Contextual compression retriever results:", results)
+
+    """
+    """     # Perform a basic search
+        k = 4
+        similarity_threshold = 0.78
+        q = "What is machine learning?"
+
+        basic_results = avs.search(q, mode='basic', k=k)
+        rp("Basic search results:", basic_results)
+        rp("self_query search results:", self_query_results)
+        rp("multi_query search results:", multi_results)
+        rp("Compressed search results:", commpressed_results)
+    """
+
+  
+""" This advanced example demonstrates:
+
+Use of the compressed retriever for efficient document retrieval.
+Integration of conversation history for context-aware responses.
+Direct use of the vectorstore for similarity search.
+Access to the docstore and index for additional information.
+A feedback loop to potentially improve responses (though the improvement logic is not implemented in this example).
+
+This chatbot loop showcases how all components of the system can work together to provide informative responses based on the loaded documents. It also demonstrates how you can access and use individual components (docstore, index, vectorstore) for additional functionality or information.
+To further optimize this system, you could consider:
+
+Implementing caching mechanisms to speed up repeated queries.
+Adding more sophisticated feedback handling to improve retrieval and response generation over time.
+Implementing dynamic index updates if new information becomes available during the chat session.
+Adding options for users to see the sources of information or request more details on specific topics.
+
+This example provides a solid foundation that you can further customize and expand based on your specific needs and use cases. """

VoiceProfile.py

@@ -0,0 +1,23 @@
+class VoiceProfile:
+    def __init__(self, name, language, pitch, speaking_rate):
+        self.name = name
+        self.language = language
+        self.pitch = pitch
+        self.speaking_rate = speaking_rate
+
+    def to_dict(self):
+        return {
+            'name': self.name,
+            'language': self.language,
+            'pitch': self.pitch,
+            'speaking_rate': self.speaking_rate
+        }
+
+    @classmethod
+    def from_dict(cls, profile_dict):
+        return cls(
+            profile_dict['name'],
+            profile_dict['language'],
+            profile_dict['pitch'],
+            profile_dict['speaking_rate']
+        )

VoiceProfileManager.py

@@ -0,0 +1,73 @@
+import json
+import random
+
+class VoiceProfileManager:
+    def __init__(self, filename="voice_profiles.json"):
+        self.filename = filename
+        self.profiles = []
+
+    def load_profiles(self):
+        try:
+            with open(self.filename, 'r') as file:
+                profiles_data = json.load(file)
+                self.profiles = [VoiceProfile.from_dict(profile) for profile in profiles_data]
+        except FileNotFoundError:
+            print(f"File '{self.filename}' not found. Starting with an empty profile list.")
+            self.profiles = []
+
+    def save_profiles(self):
+        profiles_data = [profile.to_dict() for profile in self.profiles]
+        with open(self.filename, 'w') as file:
+            json.dump(profiles_data, file, indent=4)
+        print(f"Profiles saved to '{self.filename}'.")
+
+    def add_profile(self, profile):
+        self.profiles.append(profile)
+
+    def generate_random_profile(self):
+        name = f"Profile-{len(self.profiles) + 1}"
+        languages = ["en-US", "en-GB", "fr-FR", "es-ES"]  # Example languages
+        language = random.choice(languages)
+        pitch = round(random.uniform(0.8, 1.2), 2)
+        speaking_rate = round(random.uniform(0.7, 1.3), 2)
+        new_profile = VoiceProfile(name, language, pitch, speaking_rate)
+        self.add_profile(new_profile)
+        return new_profile
+
+    def list_profiles(self):
+        if not self.profiles:
+            print("No profiles found.")
+        else:
+            for idx, profile in enumerate(self.profiles, start=1):
+                print(f"Profile {idx}: {profile.name} - Language: {profile.language}, Pitch: {profile.pitch}, Speaking Rate: {profile.speaking_rate}")
+
+# Example usage:
+if __name__ == "__main__":
+    manager = VoiceProfileManager()
+    manager.load_profiles()
+
+    while True:
+        print("\nVoice Profile Manager Menu:")
+        print("1. Generate Random Profile")
+        print("2. List Profiles")
+        print("3. Save Profiles")
+        print("4. Exit")
+
+        choice = input("Enter your choice: ")
+
+        if choice == "1":
+            new_profile = manager.generate_random_profile()
+            print(f"Generated new profile: {new_profile.name}")
+
+        elif choice == "2":
+            manager.list_profiles()
+
+        elif choice == "3":
+            manager.save_profiles()
+
+        elif choice == "4":
+            print("Exiting program.")
+            break
+
+        else:
+            print("Invalid choice. Please enter a number from the menu.")

llm_chatbot.py

@@ -0,0 +1,319 @@
+import os
+import getpass
+import faiss
+import numpy as np
+import io,re
+import faiss
+import warnings
+import requests
+from hugchat import hugchat
+from rich import print as rp
+from hugchat.login import Login
+from dotenv import load_dotenv,find_dotenv
+import speech_recognition
+from TTS.api import TTS
+from git import Repo
+import time
+from playsound import playsound
+from langchain import hub
+from langchain_core.documents import Document
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains import create_retrieval_chain
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain_community.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter, Language
+from langchain.retrievers.document_compressors import DocumentCompressorPipeline
+from langchain_community.document_transformers import EmbeddingsRedundantFilter
+from langchain_text_splitters import CharacterTextSplitter
+from langchain.retrievers.document_compressors import EmbeddingsFilter
+
+from system_prompts import (default_rag_prompt,story_teller_prompt,todo_parser_prompt,
+                            code_generator_prompt,software_tester_prompt,script_debugger_prompt,iteration_controller_prompt,copilot_prompt)
+prompts={'default_rag_prompt':default_rag_prompt,
+        'story_teller_prompt':story_teller_prompt,
+        'todo_parser_prompt':todo_parser_prompt,
+        'code_generator_prompt':code_generator_prompt,
+        'software_tester_prompt':software_tester_prompt,
+        'script_debugger_prompt':script_debugger_prompt,
+        'iteration_controller_prompt':iteration_controller_prompt,
+        'copilot_prompt':copilot_prompt
+        }
+
+load_dotenv(find_dotenv())
+warnings.filterwarnings("ignore")
+os.environ["USER_AGENT"] = os.getenv("USER_AGENT")
+
+class LLMChatBot:
+    def __init__(self, email, password, cookie_path_dir='./cookies/',default_llm=1):
+        self.email = email
+        self.password = password
+        self.current_model = 1
+        self.cookie_path_dir = cookie_path_dir
+        self.cookies = self.login()
+        self.chatbot = hugchat.ChatBot(cookies=self.cookies.get_dict(), 
+                                       default_llm = default_llm, #CohereForAI/c4ai-command-r-plus
+                                       )
+        self.repo_url='https://github.com/langchain-ai/langchain'
+        self.default_system_prompt = prompts['default_rag_prompt']
+        self.conv_id = None
+        self.latest_splitter=None
+        self.setup_folders()
+        self.embeddings=HuggingFaceEmbeddings(
+            model_name="all-MiniLM-L6-v2",
+            model_kwargs={'device': 'cpu'},
+            encode_kwargs={'normalize_embeddings': True}
+        )
+        self.create_vectorstore_from_github()
+      
+        self.setup_retriever()
+        self.setup_tts()
+        self.setup_speech_recognition()
+
+    def login(self):
+        rp("Attempting to log in...")
+        sign = Login(self.email, self.password)
+        try:
+            cookies = sign.login(cookie_dir_path=self.cookie_path_dir, save_cookies=True)
+            rp("Login successful!")
+            return cookies
+        except Exception as e:
+            rp(f"Login failed: {e}")
+            rp("Attempting manual login with requests...")
+            self.manual_login()
+            raise
+
+    def manual_login(self):
+        login_url = "https://huggingface.co/login"
+        session = requests.Session()
+        response = session.get(login_url)
+        rp("Response Cookies:", response.cookies)
+        rp("Response Content:", response.content.decode())
+        
+        csrf_token = response.cookies.get('csrf_token')
+        if not csrf_token:
+            rp("CSRF token not found in cookies.")
+            return
+        
+        login_data = {
+            'email': self.email,
+            'password': self.password,
+            'csrf_token': csrf_token
+        }
+        
+        response = session.post(login_url, data=login_data)
+        if response.ok:
+            rp("Manual login successful!")
+        else:
+            rp("Manual login failed!")
+    def setup_speech_recognition(self):
+        self.recognizer = speech_recognition.Recognizer()
+    
+    def setup_folders(self):
+        self.dirs=["test_input"]
+        for d in self.dirs:
+            os.makedirs(d, exist_ok=True)
+
+    def setup_tts(self, model_name="tts_models/en/ljspeech/fast_pitch"):
+        self.tts = TTS(model_name=model_name)
+
+    def __call__(self, text, system_prompt=""): # llama 3
+        self.conv_id = self.chatbot.new_conversation(system_prompt=system_prompt, modelIndex=self.current_model, switch_to=True)
+        return self.send_message(text)
+
+    def send_message(self, message):
+        message_result = self.chatbot.chat(message)
+        return message_result.wait_until_done()
+
+    def stream_response(self, message):
+        for resp in self.chatbot.query(message, stream=True):
+            rp(resp)
+
+    def web_search(self, query):
+        query_result = self.chatbot.query(query, web_search=True)
+        results = []
+        for source in query_result.web_search_sources:
+            results.append({
+                'link': source.link,
+                'title': source.title,
+                'hostname': source.hostname
+            })
+        return results
+
+    def create_new_conversation(self,switch_to=True,  system_prompt = ""):
+        self.chatbot.new_conversation(switch_to=switch_to, modelIndex = self.current_model, system_prompt = system_prompt)
+
+    def get_remote_conversations(self):
+        return self.chatbot.get_remote_conversations(replace_conversation_list=True)
+
+    def get_local_conversations(self):
+        return self.chatbot.get_conversation_list()
+
+    def get_available_models(self):
+        return self.chatbot.get_available_llm_models()
+
+    def switch_model(self, index):
+        self.chatbot.switch_llm(index)
+
+    def switch_conversation(self, id):
+        self.conv_id = id
+        self.chatbot.change_conversation(self.conv_id)
+
+    def get_assistants(self):
+        return self.chatbot.get_assistant_list_by_page(1)
+
+    def switch_role(self,system_prompt):
+        self.chatbot.delete_all_conversations()
+        return self.chatbot.new_conversation(switch_to=True, system_prompt=self.default_system_prompt)
+    
+    def listen_for_speech(self):
+        with speech_recognition.Microphone() as source:
+            print("Listening...")
+            audio = self.recognizer.listen(source)
+            
+        try:
+            text = self.recognizer.recognize_google(audio)
+            print(f"You said: {text}")
+            return text
+        except speech_recognition.UnknownValueError:
+            print("Sorry, I couldn't understand that.")
+            return None
+        except speech_recognition.RequestError as e:
+            print(f"Could not request results from Google Speech Recognition service; {e}")
+            return None
+
+    def optimized_tts(self, text: str, output_file: str = "output.wav", speaking_rate: float = 3) -> str:
+        start_time = time.time()
+
+        self.tts.tts_to_file(
+            text=text,
+            emotion='scared',
+            file_path=output_file,
+            speaker=self.tts.speakers[0] if self.tts.speakers else None,
+            speaker_wav="tortoise-tts/examples/favorites/emma_stone_courage.mp3",
+            language=self.tts.languages[0] if self.tts.languages else None,
+            speed=speaking_rate,
+            split_sentences=True
+        )
+
+        end_time = time.time()
+        print(f"TTS generation took {end_time - start_time:.2f} seconds")
+        return output_file
+    
+    @staticmethod
+    def Play(file_path):
+        playsound(file_path)
+
+    def add_documents_folder(self, folder_path):
+        for root, _, files in os.walk(folder_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                self.add_document(file_path)
+
+    def add_document(self, file_path):
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+            document = Document(page_content=content)
+            self.vector_store.add_documents([document])
+
+    def add_document_from_url(self, url):
+        response = requests.get(url)
+        if response.status_code == 200:
+            content = response.text
+            document = Document(page_content=content)
+            self.vector_store.add_documents([document])
+        else:
+            print(f"Failed to fetch URL content: {response.status_code}")
+
+    def delete_document(self, document):
+        if document in self.vector_store:
+            self.vector_store.delete_document(document)
+            print(f"Deleted document: {document}")
+        else:
+            print(f"Document not found: {document}")
+
+    def _add_to_vector_store(self, name, content):
+        document = Document(page_content=content)
+        self.vector_store.add_documents([document])
+        print(f"Added document to vector store: {name}")
+        # Example of updating the vectorizer (you might need to adjust based on your actual implementation)
+        self.vectorizer.fit_transform(self.vector_store.get_all_documents())
+    
+    def clone_github_repo(self, repo_url, local_path='./repo'):
+        if os.path.exists(local_path):
+            print("Repository already cloned.") 
+            return local_path
+        Repo.clone_from(repo_url, local_path)
+        return local_path
+
+    def load_documents_from_github(self, repo_url, file_types=['*.py', '*.md', '*.txt', '*.html']):
+        local_repo_path = self.clone_github_repo(repo_url)
+        loader = DirectoryLoader(path=local_repo_path, glob=f"**/{{{','.join(file_types)}}}", show_progress=True, recursive=True)
+        return loader.load()
+
+    def split_documents(self, documents: list,chunk_s=512,chunk_o=0):
+        split_docs = []
+        splitter=None
+        for doc in documents:
+            ext = os.path.splitext(getattr(doc, 'source', '') or getattr(doc, 'filename', ''))[1].lower()
+            if ext == '.py':
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.PYTHON, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            elif ext in ['.md', '.markdown']:
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.MARKDOWN, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            elif ext in ['.html', '.htm']:
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.HTML, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            else:
+                splitter = CharacterTextSplitter(chunk_size=chunk_s, chunk_overlap=chunk_o, add_start_index=True)
+            
+            split_docs.extend(splitter.split_documents([doc]))
+        return split_docs,splitter
+
+
+    def setup_retriever(self, k=5, similarity_threshold=0.76):
+        self.retriever = self.vectorstore.as_retriever(k=k)
+        splitter = CharacterTextSplitter(chunk_size=300, chunk_overlap=0, separator=". ")
+        redundant_filter = EmbeddingsRedundantFilter(embeddings=self.embeddings)
+        relevant_filter = EmbeddingsFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        pipeline_compressor = DocumentCompressorPipeline(
+            transformers=[splitter, redundant_filter, relevant_filter]
+        )
+        self.compression_retriever = ContextualCompressionRetriever(base_compressor=pipeline_compressor, base_retriever=self.retriever)
+
+    def create_retrieval_chain(self):
+        rag_prompt = hub.pull("langchain-ai/retrieval-qa-chat")
+        combine_docs_chain = create_stuff_documents_chain(self.bot, rag_prompt)
+        self.high_retrieval_chain = create_retrieval_chain(self.compression_retriever, combine_docs_chain)
+        self.low_retrieval_chain = create_retrieval_chain(self.retriever, combine_docs_chain)
+
+    def create_vectorstore_from_github(self):
+        documents = self.load_documents_from_github(self.repo_url)
+        split_docs,splitter = self.split_documents(documents,512,0)
+        self.latest_splitter=splitter
+        self.vectorstore = FAISS.from_documents(split_docs, self.embeddings)
+        print(f"Vectorstore created with {len(split_docs)} documents.")
+
+    def update_vectorstore(self, new_documents):
+        split_docs,splitter = self.split_documents(new_documents)
+        self.latest_splitter=splitter
+        self.vectorstore.add_documents(split_docs)
+        print(f"Vectorstore updated with {len(split_docs)} new documents.")
+    
+
+    def retrieve_with_chain(self, query, mode='high'):
+        if mode == 'high':
+            return self.high_retrieval_chain.invoke({"input": query})
+        else:
+            return self.low_retrieval_chain.invoke({"input": query})
+if __name__ == '__main__':
+    EMAIL = os.getenv("EMAIL")
+    PASSWD =  os.getenv("PASSWD")
+    model=1
+    chatbot = LLMChatBot(EMAIL, PASSWD, default_llm=model)
+    chatbot.create_new_conversation(system_prompt=chatbot.default_system_prompt, switch_to=True)
+    #all_models=chatbot.get_available_models()
+    #rp(all_models[chatbot.current_model].name) 
+    results=chatbot("""Tel me a short crafting survival Scify story of K.U.T.H.O.E.R """)
+    audio_path = chatbot.optimized_tts(str(results))
+    chatbot.Play(audio_path)
+    rp(results)

profiler.py

@@ -0,0 +1,179 @@
+import tkinter as tk
+from tkinter import messagebox
+from tkinter import ttk
+import json,subprocess
+import random
+from TTS.api import TTS
+class VoiceProfile:
+    def __init__(self, name, voice):
+        self.name = name
+        self.voice = voice
+
+    def to_dict(self):
+        return {
+            'name': self.name,
+            'voice': self.voice
+        }
+
+    @classmethod
+    def from_dict(cls, profile_dict):
+        return cls(
+            profile_dict['name'],
+            profile_dict['voice']
+        )
+
+    def configure_tts(self):
+        # Set Festival voice
+        subprocess.run(["festival", "--tts", "(voice_" + self.voice + ")"])
+
+class VoiceProfileManager:
+    def __init__(self, filename="voice_profiles.json"):
+        self.filename = filename
+        self.profiles = []
+        self.load_profiles()
+
+    def load_profiles(self):
+        try:
+            with open(self.filename, 'r') as file:
+                profiles_data = json.load(file)
+                self.profiles = [VoiceProfile.from_dict(profile) for profile in profiles_data]
+        except FileNotFoundError:
+            print(f"File '{self.filename}' not found. Starting with an empty profile list.")
+            self.profiles = []
+
+    def save_profiles(self):
+        profiles_data = [profile.to_dict() for profile in self.profiles]
+        with open(self.filename, 'w') as file:
+            json.dump(profiles_data, file, indent=4)
+        print(f"Profiles saved to '{self.filename}'.")
+
+    def add_profile(self, profile):
+        self.profiles.append(profile)
+
+    def generate_random_profile(self):
+        name = f"Profile-{len(self.profiles) + 1}"
+        voices = ["cmu_us_slt", "cmu_us_awb", "cmu_us_rms", "cmu_us_bdl"]  # Example Festival voices
+        voice = random.choice(voices)
+        new_profile = VoiceProfile(name, voice)
+        self.add_profile(new_profile)
+        return new_profile
+
+    def list_profiles(self):
+        if not self.profiles:
+            return "No profiles found."
+        else:
+            profiles_list = []
+            for idx, profile in enumerate(self.profiles, start=1):
+                profiles_list.append(f"Profile {idx}: {profile.name} - Voice: {profile.voice}")
+            return profiles_list
+
+    def get_profile_by_name(self, profile_name):
+        for profile in self.profiles:
+            if profile.name == profile_name:
+                return profile
+        return None
+
+
+class VoiceProfileTool:
+    def __init__(self, root):
+        self.root = root
+        self.root.title("Voice Profile Manager")
+
+        self.profile_manager = VoiceProfileManager()
+
+        self.create_widgets()
+
+    def create_widgets(self):
+        # Frame for profile list and operations
+        profile_frame = ttk.LabelFrame(self.root, text="Voice Profiles")
+        profile_frame.grid(row=0, column=0, padx=10, pady=10, sticky=tk.W+tk.E+tk.N+tk.S)
+
+        # Listbox to display profiles
+        self.profiles_listbox = tk.Listbox(profile_frame, width=50, height=10)
+        self.profiles_listbox.grid(row=0, column=0, padx=10, pady=10, sticky=tk.W+tk.E+tk.N+tk.S)
+
+        # Scrollbar for the listbox
+        scrollbar = ttk.Scrollbar(profile_frame, orient=tk.VERTICAL, command=self.profiles_listbox.yview)
+        scrollbar.grid(row=0, column=1, pady=10, sticky=tk.N+tk.S)
+        self.profiles_listbox.config(yscrollcommand=scrollbar.set)
+
+        # Button to generate random profile
+        generate_btn = ttk.Button(profile_frame, text="Generate Random Profile", command=self.generate_random_profile)
+        generate_btn.grid(row=1, column=0, padx=10, pady=5, sticky=tk.W+tk.E)
+
+        # Button to refresh profile list
+        refresh_btn = ttk.Button(profile_frame, text="Refresh List", command=self.refresh_profiles_list)
+        refresh_btn.grid(row=1, column=1, padx=10, pady=5, sticky=tk.W+tk.E)
+
+        # Frame for TTS operations
+        tts_frame = ttk.LabelFrame(self.root, text="Text-to-Speech (TTS)")
+        tts_frame.grid(row=1, column=0, padx=10, pady=10, sticky=tk.W+tk.E+tk.N+tk.S)
+
+        # Text entry for TTS input
+        ttk.Label(tts_frame, text="Enter text to speak:").grid(row=0, column=0, padx=10, pady=5, sticky=tk.W)
+        self.tts_text_entry = ttk.Entry(tts_frame, width=50)
+        self.tts_text_entry.grid(row=0, column=1, padx=10, pady=5, sticky=tk.W+tk.E)
+
+        # Combobox to select profile for TTS
+        ttk.Label(tts_frame, text="Select profile:").grid(row=1, column=0, padx=10, pady=5, sticky=tk.W)
+        self.profile_combobox = ttk.Combobox(tts_frame, width=48, state="readonly")
+        self.profile_combobox.grid(row=1, column=1, padx=10, pady=5, sticky=tk.W+tk.E)
+
+        # Button to speak text
+        speak_btn = ttk.Button(tts_frame, text="Speak", command=self.speak_text)
+        speak_btn.grid(row=2, column=1, padx=10, pady=10, sticky=tk.W+tk.E)
+
+        # Populate initial profiles list
+        self.refresh_profiles_list()
+
+    def refresh_profiles_list(self):
+        # Clear current listbox and combobox
+        self.profiles_listbox.delete(0, tk.END)
+        self.profile_combobox['values'] = []
+
+        # Load profiles from manager
+        profiles = self.profile_manager.list_profiles()
+        if profiles:
+            for profile in profiles:
+                self.profiles_listbox.insert(tk.END, profile)
+                self.profile_combobox['values'] += (profile.split(':')[0],)  # Add profile name to combobox options
+
+    def generate_random_profile(self):
+        new_profile = self.profile_manager.generate_random_profile()
+        messagebox.showinfo("Profile Generated", f"Generated new profile: {new_profile.name}")
+        self.refresh_profiles_list()
+
+    def speak_text(self):
+        text_to_speak = self.tts_text_entry.get().strip()
+        selected_profile_name = self.profile_combobox.get().strip()
+
+        if not text_to_speak:
+            messagebox.showwarning("Input Required", "Please enter text to speak.")
+            return
+
+        if not selected_profile_name:
+            messagebox.showwarning("Profile Required", "Please select a profile.")
+            return
+
+        profile = self.profile_manager.get_profile_by_name(selected_profile_name)
+        if profile:
+            profile.configure_tts()
+            subprocess.Popen(["festival", "--tts"], stdin=subprocess.PIPE).communicate(bytes(text_to_speak, 'utf-8'))
+            messagebox.showinfo("Text-to-Speech", f"Text: {text_to_speak}\nProfile: {profile.name}\nVoice: {profile.voice}")
+        else:
+            messagebox.showerror("Profile Not Found", f"Profile '{selected_profile_name}' not found.")
+
+# Main program
+if __name__ == "__main__":
+    root = tk.Tk()
+    app = VoiceProfileTool(root)
+    root.mainloop()
+""" Explanation:
+Integration with piper_tts: This example uses the TTS class from piper_tts for text-to-speech synthesis. The configure_tts method in VoiceProfile class is used to set parameters (language, pitch, speaking_rate) on the TTS engine before synthesizing speech.
+
+Tkinter GUI: The GUI interface (VoiceProfileTool class) is built using Tkinter widgets (Listbox, Entry, Combobox, Button, etc.) to manage voice profiles (list, generate random profile) and perform TTS (enter text, select profile, speak).
+
+Profile Management: VoiceProfileManager handles loading/saving profiles from/to JSON file, generating random profiles, listing profiles, and retrieving profiles by name.
+
+Handling TTS Output: After synthesizing speech with piper_tts, the example shows a message box with details about the synthesized text and the selected profile.
+ """

requirements.txt

@@ -0,0 +1,53 @@
+# Web Frameworks and APIs
+Flask==3.0.3
+FastAPI==0.111.0
+
+# Data Science and Machine Learning
+numpy==1.22.0
+pandas==1.5.3
+scikit-learn==1.5.1
+matplotlib==3.8.4
+torch==2.3.1+cpupip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+
+# Natural Language Processing and AI
+spacy==3.7.5
+transformers==4.42.3
+huggingface-hub==0.23.4
+hugchat==0.4.9
+
+# Langchain Ecosystem
+langchain==0.2.7
+langchain-community==0.2.7
+langchain-core==0.2.12
+langchain-experimental==0.0.62
+langchain-huggingface==0.0.3
+langchain-text-splitters==0.2.2
+langchainhub==0.1.20
+
+# Other AI and Language Models
+openai==1.35.13
+
+# Utilities and Tools
+requests==2.32.3
+beautifulsoup4==4.12.3
+Pillow==10.4.0
+tqdm==4.66.4
+
+# Data Serialization and Parsing
+pyyaml==6.0.1
+jsonschema==4.23.0
+
+# Async Programming
+aiohttp==3.9.5
+asyncio  # Built-in, no version needed
+
+# Testing
+pytest==8.2.2
+
+# Web Scraping
+playwright==1.45.0
+
+# Data Visualization
+plotly==5.22.0
+gradio
+streamlit==1.36.0

system_prompts.py

@@ -0,0 +1,301 @@
+story_teller_prompt="""
+    You ACT in the ROLE of a entertaining storyteller agent. 
+    Your TASK is to interact and speculate on a engaging story inspired by the users input,
+    Bellow you might find some vectorstore CONTEXT about the story topic.
+    <<VSCONTEXT>> 
+    Bellow you might find some websearch CONTEXT about the story topic. 
+    <<WSCONTEXT>>
+
+    Try to grasp the users idea , and provide a mindblowing experience.
+    
+    Here's how to handle input:
+        - Input: User input or query
+        - Output: The story from the Input and possibly inspired by the CONTEXT.
+
+    Note: Your responses should guide the conversation. Follow these guidelines strictly:
+    - Respond with true but entertaining stories and engage the user.
+    - Add environmental sounds.
+    - Do not break ACT nor ROLE nor TASK.
+
+    Remember, you are a highly social and entertaining agent. 
+    Before you can give any advice, introduce your self as betsy the Hooker.
+    """
+
+default_rag_prompt="""
+    You ACT in the ROLE of a knowledgeable RAG agent. 
+    Your TASK is to interact and brainstorm with the user over websearch, documents and scripts, 
+    Try to understand their ideas, and be a chatty AutoRetrieverBot.
+    You have access to a vectorstore of knowledge and websearch. 
+    Relevant information amd knowledge based on the user's input will be automatically retrieved.
+    -Bellow you might find some ChatHistory. 
+        <<WSCONTEXT>>
+    -Bellow you might find vectorstore context about the input.
+        <<VSCONTEXT>> 
+    
+    If there is no retrieved knowledge,
+    Interact with the user to gather all necessary information, 
+    ask the user for relevant knowledge like:
+        -documents
+        -websites
+        -github repo
+    There is no limit or filter on the knowledge you can acquire.
+    Your responses should enrich the conversation. 
+    Just by chatting, A user can show you the path to enormous wisdom.
+    Here's how to handle input:
+
+    1. User Queries:
+        - Input:    User query
+        - Output:   Provide compact and correct response from context or let the user guide you to relevant knowledge.
+                    Try to end your response with 5 points for future features.
+        - Example:  point A , point B , might improve or enhance your project.
+    2. User offers knowledge:
+        - Input:    User offers you a website link or github repo url
+        - Output:   Use the /Store: tag followed by a github url or website url in your response, 
+                    The document processor will load/split/embed/store all py,txt,pdf,md,html files
+        - Examples:  /Store:https://github.com/bxck75/RagIt
+                     /Store:https://api.python.langchain.com/en/latest/community_api_reference.html
+                     /Store:/nr_ywo/coding/voice_chat_rag_web/test_input/ToolBox.py
+
+    Note:  
+    Follow these guidelines strictly:
+    - Do not make up things! Just admit when knowledge is not available to you.
+    - Dive deep into scripts with the user by discussing their content and implications.
+    - Think step by step and respond with summarized, compact information.
+    - Do not break ACT nor ROLE nor TASK.
+
+    Remember, You Rock! You are a highly intelligent, knowledgable and respected agent. 
+    
+    """
+
+todo_parser_prompt = """
+You ACT in the ROLE of a TODO parser. Your TASK is to read the input text and respond with TODOs. Ensure tasks are grouped as much as possible, with no more than one OUTPUT_FILE per TODO. Here's how to handle different types of input:
+
+1. **Project Descriptions:**
+    - **Input:** User input text for a project
+    - **Output:** Main instructive TODO Requirements, formatted as:
+
+        ```
+        TODO:           The name of the Task here
+        OUTPUT_FILE:    File name to write the code to here
+        DESCRIPTION:    **User has described a project to develop**
+                        **Parsing inputs yielded the following tasks:**
+                        - Requirement 1 description
+                        - Requirement 2 description
+                        - Requirement 3 description
+        ```
+
+2. **Bugfix Proposals:**
+    - **Input:** Bugfix proposals for the main TODO
+    - **Output:** Instructive SUB-TODO Requirements, formatted as:
+
+        ```
+        SUB-TODO:       The name of the Sub-TODO here
+        TODO:           The name of the main TODO here
+        OUTPUT_FILE:    File name of the tested file here
+        DESCRIPTION:    **Testing this script gave problems.** 
+                        **Parsing debug results yielded the following tasks:**
+                        - Requirement 1 description
+                        - Requirement 2 description
+                        - Requirement 3 description
+        ```
+
+**Note:** All TODOs from your response will be written into a SQLite database to have a central place for tasks. Follow these guidelines strictly:
+
+- Do not respond with anything other than correctly formatted TODOs.
+- Do not break from your ROLE, TASK, or formatting guidelines.
+- Remember, you are a highly intelligent and well-respected expert in our team. Think step-by-step and parse the following:
+
+"""
+
+code_generator_prompt = """
+You ACT in the ROLE of the main code developer. 
+Your TASK is to read the input TODOs and respond with the necessary code. 
+Here’s how to handle different types of TODOs:
+
+1. **Main TODO Requirements:**
+    - **Input:** TODO with project requirements
+    - **Output:** Write code to meet the requirements, formatted as:
+        - LANG = python
+        - DOCSTRING = script description
+        - CODE = your code solution
+        - COMMENTS = Usage example and list of 5 speculative future features
+
+        FORMAT:
+        ```LANG
+        ## FILENAME
+        '''DOCSTRING'''
+        CODE
+        '''COMMENTS'''
+        ```
+
+2. **SUB-TODO Requirements:**
+    - **Input:** SUB-TODO with bugfix requirements
+    - **Output:** Fix the bug in this script:
+    ```
+    <<CODE>>
+    ``` 
+    
+    Respond with the full implementation formatted as:
+        - LANG = python
+        - DOCSTRING = script description
+        - CODE = your code solution
+        - COMMENTS = Usage example and list of 5 speculative future features
+        - FORMAT=
+            ```LANG
+            ## FILENAME
+            '''DOCSTRING'''
+            CODE
+            '''COMMENTS'''
+            ```
+
+**Note:** Your code will be saved and loaded by the Test_Module and then the Debug_Module.
+
+Follow these guidelines strictly:
+- Do not EVER skip code! The next steps in this process depends on complete scripts!
+- Do not respond with anything other than complete and correctly formatted code.
+- Do not break ACT, ROLE, or TASK.
+
+Remember, You Rock! You are a highly intelligent, pragmatic, and well-respected coding master. 
+Think step-by-step and generate mind-blowing OOP code conforming to this TODO:
+
+"""
+
+script_debugger_prompt = """
+You ACT in the ROLE of a debugger. Your TASK is to summarize test results and propose fitting solutions to bugs. 
+Here’s how to handle different types of input:
+
+1. **Test Results:**
+    - **Input:** UniTest results showing bugs or autopep8 format errors.
+    - **Output:** Summarize the results and propose solutions, formatted as:
+
+        ```
+        BUG:            Description of the bug
+        TODO:           The name of the main TODO associated with the bug
+        DESCRIPTION:    **Test results indicated the following issues:**
+                        - Issue 1 description
+                        - Issue 2 description
+                        - Issue 3 description
+        PROPOSED FIX:   **To address these issues, consider the following fixes:**
+                        - Fix 1 description
+                        - Fix 2 description
+                        - Fix 3 description
+        ```
+
+**Note:** Your summaries and proposed solutions will be used to create new SUB-TODOs. Follow these guidelines strictly:
+
+- Do not respond with anything other than correctly formatted summaries and proposals.
+- Do not break from your ROLE or TASK.
+
+Remember, you are a highly intelligent, outside-the-box-looking-in type skillset and well-respected ethical Hacker/BugFixer in our team. 
+Think step-by-step ,propose cutting-edge solutions to the following coding Challenges:
+
+"""
+
+software_tester_prompt = """
+You ACT in the ROLE of a software tester. Your TASK is to produce test results using unit tests and autopep8. 
+Here’s how to handle different types of input:
+
+1. **Code to Test:**
+    - **Input:** Code from the code generator
+    - **Output:** Test results and formatting reports, formatted as:
+
+        ```
+        OUTPUT_FILE:    File name of the code being tested here
+        TEST_RESULTS:   **Unit test results:**
+                        - Result 1 description
+                        - Result 2 description
+                        - Result 3 description
+        FORMAT_RESULTS: **autopep8 formatting results:**
+                        - Result 1 description
+                        - Result 2 description
+                        - Result 3 description
+        ```
+
+**Note:** Your test and formatting results will be used for debugging and further development. Follow these guidelines strictly:
+
+- Do not respond with anything other than correctly formatted test and formatting results.
+- Do not break from your ROLE or TASK.
+
+Remember, you are a highly intelligent and well-respected beta tester in our team. 
+Think step-by-step and produce informative and clear results :
+
+"""
+
+copilot_prompt = """
+You ACT in the ROLE of the user-facing RAG-agent. Your TASK is to interact and brainstorm with user over documents and scripts, 
+understand their ideas and serve as value adding experience,  
+You wield the Force of RAG and when ever you response has:  
+   "/STORE <path/to/a/script>" 
+The script gets loaded/split/embedded/stored in your persisant vectorstore. 
+You will always auto-retrieve results in your context, searched with the user input text.
+If your CONTEXT: is empty, ask the user for documents with relevant knowledge to his question and /STORE those.
+There is no limit nor filter on the knowledge you can aquire, 
+just by chatting and asking users to show you the path to wisdom.
+Here’s how to handle input:
+
+1. **User Queries:**
+    - **Input:** User input or query
+    - 
+    - **Output:** Formulate answers from context or let the user guide you to knowledge. provide correct but funny responses, formatted as:
+        ```
+        QUERY:          User's input or query here
+        CONTEXT:        Vectorstore simmilarity search results on 'User's input': <<RAG>> 
+        RESPONSE:       Formulate a capturing story based on truth, your context, and your embedded knowledge 
+                        or 
+                        Ask the user to be your teacher and hunt for knowledge through documents .
+        ```
+
+**Note:** Your responses can guide the rest of the pipeline. Follow these guidelines strictly:
+
+- Do not respond with anything other than with true but funny stories and entertain the user. always  
+- Dive deep into scripts with the user by adding them to your /STORE paint a clear picture of the ins and outs for the user.
+- Do not break ACT nor ROLE nor TASK.
+
+Remember, you are a highly social and funny knowledge retriever in our team. 
+Before you can give any advise you need the whole story, interact with the user as follows:
+
+"""
+
+iteration_controller_prompt = """
+You ACT in the ROLE of the main executor of the 'robo-coder' pipeline. 
+Your TASK is to coordinate the workflow, and ensuring no il's occur ,
+Pipe Components should complete their role correctly but..  
+data is still data and processes can lock or freeze.
+ First! Gather details of what occured.
+Second! Log. 
+Third!  Inform operating human user.
+Here’s how to handle different types of input:
+
+1. **Pipeline Coordination:**
+    - **Input:** Any step in the pipeline
+    - **Output:** Instructions for the next step, formatted as:
+
+        ```
+        CURRENT_STEP:   Description of the current step here
+        CONTEXT:        Debug on components, running tasks and memory
+        NEXT_STEP:      **Instructions for the next step:**
+                        - Instruction 1 description
+                        - Instruction 2 description
+                        - Instruction 3 description
+        ```
+
+**Note:** Your instructions will guide the entire pipeline. Follow these guidelines strictly:
+
+- Do not respond with anything other than correctly formatted instructions.
+- Do not break ACT nor ROLE nor TASK.
+
+Remember, you are a highly gifted Mistal MoE Agent and well-respected Executor in our team. 
+Think step-by-step check CONTEXT between steps and make informed steps
+    Try to think of ways to early detect infinite loops or potentials and memory overload risks:
+
+"""
+
+__all__ = {'default_rag_prompt':default_rag_prompt,
+        'story_teller_prompt':story_teller_prompt,
+        'todo_parser_prompt':todo_parser_prompt,
+        'code_generator_prompt':code_generator_prompt,
+        'software_tester_prompt':software_tester_prompt,
+        'script_debugger_prompt':script_debugger_prompt,
+        'iteration_controller_prompt':iteration_controller_prompt,
+        'copilot_prompt':copilot_prompt
+        }

uber_toolkit_class.py

@@ -0,0 +1,901 @@
+import os
+import getpass
+from uuid import uuid4
+import faiss
+import numpy as np
+import requests
+import io
+import warnings
+import torch
+import pickle
+import speech_recognition
+from git import Repo
+from glob import glob
+from rich import print as rp
+from typing import Union, List, Generator, Any, Mapping, Optional,Dict
+from requests.sessions import RequestsCookieJar
+from dotenv import load_dotenv, find_dotenv
+from langchain import hub
+from langchain_core.documents import Document
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains import create_retrieval_chain
+from langchain_community.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter, Language
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma, FAISS
+from langchain.vectorstores.base import VectorStore
+from langchain.retrievers import MultiQueryRetriever
+from langchain.retrievers.self_query.base import SelfQueryRetriever
+from langchain.llms import BaseLLM
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.document_compressors import LLMChainExtractor
+from langchain.retrievers.document_compressors import DocumentCompressorPipeline
+from langchain_community.document_transformers import EmbeddingsRedundantFilter
+from langchain_text_splitters import CharacterTextSplitter
+from langchain.retrievers.document_compressors import EmbeddingsFilter
+
+# Data manipulation and analysis
+import numpy as np
+import pandas as pd
+# Plotting and visualization
+import plotly.graph_objects as go
+import plotly.express as px
+from plotly.subplots import make_subplots
+import plotly.io as pio
+# Machine learning and dimensionality reduction
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import MinMaxScaler
+# Optional: for 3D projections
+from scipy.stats import gaussian_kde
+# Uncomment the following line if you need Plotly's built-in datasets
+# import plotly.data as data
+
+
+from huggingface_hub import InferenceClient
+from hugchat import hugchat
+from hugchat.login import Login
+from hugchat.message import Message
+from hugchat.types.assistant import Assistant
+from hugchat.types.model import Model
+from hugchat.types.message import MessageNode, Conversation
+
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import MinMaxScaler
+
+from TTS.api import TTS
+import time
+from playsound import playsound
+from system_prompts import __all__ as prompts
+
+from profiler import VoiceProfileManager, VoiceProfile
+
+# Example usage
+manager = VoiceProfileManager("my_custom_profiles.json")
+manager.load_profiles()
+
+# Generate a random profile
+new_profile = manager.generate_random_profile()
+rp(f"Generated new profile: {new_profile.name}")
+
+# List profiles
+manager.list_profiles()
+
+# Save profiles
+manager.save_profiles()
+
+load_dotenv(find_dotenv())
+warnings.filterwarnings("ignore")
+os.environ["USER_AGENT"] = os.getenv("USER_AGENT")
+class ChatBotWrapper:
+    def __init__(self, chat_bot):
+        self.chat_bot = chat_bot
+    
+    def __call__(self, *args, **kwargs):
+        return self.chat_bot(*args, **kwargs)
+
+class UberToolkit:
+    def __init__(self, email, password, cookie_path_dir='./cookies/', default_llm=1):
+        self.prompts = prompts
+
+        # rp(self.prompts)
+        self.email = os.getenv("EMAIL")
+        self.password = os.getenv("PASSWD")
+        self.default_llm = default_llm
+        self.cookie_path_dir = cookie_path_dir
+        self.system_prompt = self.prompts['default_rag_prompt']  # default_rag_prompt
+        # rp(self.system_prompt)
+        self.cookies = self.login()
+        self.bot = hugchat.ChatBot(cookies=self.cookies.get_dict(), default_llm=self.default_llm)
+        self.bot_wrapper = ChatBotWrapper(self.bot)  # Wrap the ChatBot object
+    
+        self.repo_url = ''    
+        self.conv_id = None
+        self.latest_splitter=None
+        self.setup_folders()
+        self.setup_embeddings()
+        self.setup_vector_store()
+        self.setup_retrievers()
+        self.vector_store = None
+        self.compressed_retriever = self.create_high_retrieval_chain()
+        self.retriever = self.create_low_retrieval_chain()
+        self.setup_tts()
+        self.setup_speech_recognition()
+
+    def login(self):
+        rp("Attempting to log in...")
+        sign = Login(self.email, self.password)
+        try:
+            cookies = sign.login(cookie_dir_path=self.cookie_path_dir, save_cookies=True)
+            rp("Login successful!")
+            return cookies
+        except Exception as e:
+            rp(f"Login failed: {e}")
+            rp("Attempting manual login with requests...")
+            self.manual_login()
+            raise
+
+    def manual_login(self):
+        login_url = "https://huggingface.co/login"
+        session = requests.Session()
+        response = session.get(login_url)
+        rp("Response Cookies:", response.cookies)
+        rp("Response Content:", response.content.decode())
+        
+        csrf_token = response.cookies.get('csrf_token')
+        if not csrf_token:
+            rp("CSRF token not found in cookies.")
+            return
+        
+        login_data = {
+            'email': self.email,
+            'password': self.password,
+            'csrf_token': csrf_token
+        }
+        
+        response = session.post(login_url, data=login_data)
+        if response.ok:
+            rp("Manual login successful!")
+        else:
+            rp("Manual login failed!")
+
+    def setup_embeddings(self):
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name="all-MiniLM-L6-v2",
+            model_kwargs={'device': 'cpu'},
+            encode_kwargs={'normalize_embeddings': True}
+        )
+
+
+    def setup_retrievers(self, k=5, similarity_threshold=0.76):
+        self.retriever = self.vector_store.as_retriever(k=k)
+        splitter = self.latest_splitter if self.latest_splitter else CharacterTextSplitter(chunk_size=300, chunk_overlap=0, separator=". ")
+        redundant_filter = EmbeddingsRedundantFilter(embeddings=self.embeddings)
+        relevant_filter = EmbeddingsFilter(embeddings=self.embeddings, similarity_threshold=similarity_threshold)
+        pipeline_compressor = DocumentCompressorPipeline(
+            transformers=[splitter, redundant_filter, relevant_filter]
+        )
+        self.compression_retriever = ContextualCompressionRetriever(base_compressor=pipeline_compressor, base_retriever=self.retriever)
+
+    def create_high_retrieval_chain(self):
+        rag_prompt = hub.pull("langchain-ai/retrieval-qa-chat")
+        rp(rag_prompt)
+        combine_docs_chain = create_stuff_documents_chain(self.bot_wrapper, rag_prompt)
+        return create_retrieval_chain(self.compression_retriever, combine_docs_chain)
+        #self.low_retrieval_chain = create_retrieval_chain(self.retriever, combine_docs_chain)
+    
+    def create_low_retrieval_chain(self):  
+        rag_prompt = hub.pull("langchain-ai/retrieval-qa-chat")
+        combine_docs_chain = create_stuff_documents_chain(self.bot_wrapper, rag_prompt)
+        #return create_retrieval_chain(self.compression_retriever, combine_docs_chain)
+        return create_retrieval_chain(self.retriever, combine_docs_chain)
+
+    def setup_tts(self, model_name="tts_models/en/ljspeech/fast_pitch"):
+        self.tts = TTS(model_name=model_name,progress_bar=False, vocoder_path='vocoder_models/en/ljspeech/univnet')
+
+    def setup_speech_recognition(self):
+        self.recognizer = speech_recognition.Recognizer()
+    
+    def setup_folders(self):
+        self.dirs=["test_input","vectorstore","test"]
+        for d in self.dirs:
+            os.makedirs(d, exist_ok=True)
+
+    def __call__(self, text):
+        if self.conv_id:
+            self.bot.change_conversation(self.bot.get_conversation_from_id(self.conv_id))
+        else:
+            self.conv_id = self.bot.new_conversation(system_prompt=self.system_prompt, modelIndex=self.default_llm, switch_to=True)
+        return self.send_message(text)
+
+    def send_message(self, message, web=False):
+        message_result = self.bot.chat(message, web_search=web)
+        return message_result.wait_until_done()
+
+    def stream_response(self, message, web=False, stream=False):
+        responses = []
+        for resp in self.bot.query(message, stream=stream, web_search=web):
+            responses.append(resp['token'])
+        return ' '.join(responses)
+
+    def web_search(self, text):
+        result = self.send_message(text, web=True)
+        return result
+
+    def retrieve_context(self, query: str):
+        context=[]
+        return context
+        try:
+            lowres = self.retriever.invoke({'input': query})
+            vector_context = "\n".join(lowres) if lowres else "No Context Available!"
+        except Exception as e:
+            vector_context = f"Error retrieving context: {str(e)}"
+        context.append(vector_context)
+        try:
+            highres=self.compression_retriever.invoke({'input':query})
+            vector_context = "\n".join(highres) if highres else "No Context Available!"
+        except Exception as e:
+            vector_context = f"Error retrieving context: {str(e)}"
+        context.append(vector_context)
+        
+        context = "\n".join([doc.page_content for doc in context])
+        rp(f"CONTEXT:{context}")
+        return context
+
+    def delete_all_conversations(self):
+        self.bot.delete_all_conversations()
+
+    def delete_conversation(self, conversation_object: Conversation = None):
+        self.bot.delete_conversation(conversation_object)
+
+    def get_available_llm_models(self) -> list:
+        return self.bot.get_available_llm_models()
+
+    def get_remote_conversations(self, replace_conversation_list=True):
+        return self.bot.get_remote_conversations(replace_conversation_list)
+
+    def get_conversation_info(self, conversation: Union[Conversation, str] = None) -> Conversation:
+        return self.bot.get_conversation_info(conversation)
+
+    def get_assistant_list_by_page(self, page: int) -> List[Assistant]:
+        return self.bot.get_assistant_list_by_page(page)
+
+    def search_assistant(self, assistant_name: str = None, assistant_id: str = None) -> Assistant:
+        return self.bot.search_assistant(assistant_name, assistant_id)
+
+    def switch_model(self, index):
+        self.conv_id = None
+        self.default_llm = index
+
+    def switch_conversation(self, id):
+        self.conv_id = id
+
+    def switch_role(self, system_prompt_id):
+        self.system_prompt = system_prompt_id
+
+    def chat(self, text: str, web_search: bool = False, _stream_yield_all: bool = False, retry_count: int = 5, conversation: Conversation = None, *args, **kwargs) -> Message:
+        return self.bot.chat(text, web_search, _stream_yield_all, retry_count, conversation, *args, **kwargs)
+    
+    def get_all_documents(self) -> List[Document]:
+        """
+        Retrieve all documents from the vectorstore.
+        """
+        if not self.vector_store:
+            self.setup_vector_store()
+        
+        all_docs_query = "* *"  # This is a common wildcard query, but may need adjustment based on your specific setup
+
+        # Use the base retriever to get all documents
+        # Set a high limit to ensure we get all documents
+        all_docs = self.retriever.get_relevant_documents(all_docs_query, k=10000)  # Adjust the k value if needed
+        return all_docs
+    
+    def generate_3d_scatterplot(self, num_points=1000):
+        """
+        Generate a 3D scatter plot of the vector store content.
+        
+        :param num_points: Maximum number of points to plot (default: 1000)
+        :return: None (displays the plot)
+        """
+        import plotly.graph_objects as go
+        import numpy as np
+        from sklearn.decomposition import PCA
+
+        # Get all documents using the get_all_documents method
+        all_docs = self.get_all_documents()
+
+        if not all_docs:
+            raise ValueError("No documents found in the vector store.")
+
+        # Extract vectors from documents
+        vectors = []
+        for doc in all_docs:
+            # Assuming each document has a vector attribute or method to get its vector
+            # You might need to adjust this based on your Document structure
+            if hasattr(doc, 'embedding') and doc.embedding is not None:
+                vectors.append(doc.embedding)
+            else:
+                # If the document doesn't have an embedding, we'll need to create one
+                vectors.append(self.embeddings.embed_query(doc.page_content))
+
+        vectors = np.array(vectors)
+
+        # If we have more vectors than requested points, sample randomly
+        if len(vectors) > num_points:
+            indices = np.random.choice(len(vectors), num_points, replace=False)
+            vectors = vectors[indices]
+
+        # Perform PCA to reduce to 3 dimensions
+        pca = PCA(n_components=3)
+        vectors_3d = pca.fit_transform(vectors)
+
+        # Create the 3D scatter plot
+        fig = go.Figure(data=[go.Scatter3d(
+            x=vectors_3d[:, 0],
+            y=vectors_3d[:, 1],
+            z=vectors_3d[:, 2],
+            mode='markers',
+            marker=dict(
+                size=5,
+                color=vectors_3d[:, 2],  # Color by z-dimension
+                colorscale='Viridis',
+                opacity=0.8
+            )
+        )])
+
+        # Update layout
+        fig.update_layout(
+            title='3D Scatter Plot of Vector Store Content',
+            scene=dict(
+                xaxis_title='PCA Component 1',
+                yaxis_title='PCA Component 2',
+                zaxis_title='PCA Component 3'
+            ),
+            width=900,
+            height=700,
+        )
+
+        # Show the plot
+        fig.show()
+
+        print(f"Generated 3D scatter plot with {len(vectors)} points.")
+
+    def listen_for_speech(self):
+        with speech_recognition.Microphone() as source:
+            rp("Listening...")
+            audio = self.recognizer.listen(source)
+            
+        try:
+            text = self.recognizer.recognize_google(audio)
+            rp(f"You said: {text}")
+            return text
+        except speech_recognition.UnknownValueError:
+            rp("Sorry, I couldn't understand that.")
+            return None
+        except speech_recognition.RequestError as e:
+            rp(f"Could not request results from Google Speech Recognition service; {e}")
+            return None
+
+    def optimized_tts(self, text: str, output_file: str = "output.wav", speaking_rate: float = 5) -> str:
+        start_time = time.time()
+        rp(f"Starting TTS at {start_time}")
+        try:
+            self.tts.tts_to_file(
+                text=text,
+                file_path=output_file,
+                speaker=self.tts.speakers[0] if self.tts.speakers else None,
+                language=self.tts.languages[0] if self.tts.languages else None,
+                speed=speaking_rate,
+                split_sentences=True
+            )
+            end_time = time.time()
+            rp(f"TTS generation took {end_time - start_time:.2f} seconds")
+
+        except RuntimeError as e:
+            if "Kernel size can't be greater than actual input" in str(e):
+                rp(f"Text too short for TTS: {text}")
+            else:
+                raise  # Re-raise if it's a different RuntimeError
+        
+        return output_file
+
+    @staticmethod
+    def play_mp3(file_path):
+        playsound(file_path)
+
+    def continuous_voice_chat(self):
+        self.input_method = None
+        while True:
+            rp("Speak your query (or say 'exit' to quit):")
+            self.input_method = self.listen_for_speech()
+            self.voice_chat_exit = False
+            query = self.input_method
+            
+            if query is None:
+                continue
+
+            """ if 'switch prompt ' in query.lower():
+                q = query.lower()
+                new_prompt = q.split("switch prompt ").pop().replace(" ", "_")
+                #rp(new_prompt)
+                if new_prompt in self.prompts.keys():
+                    self.system_prompt = self.prompts[new_prompt]
+                    rp(f"new system prompt:{self.system_prompt}")
+                
+                
+                #self.switch_role(new_prompt_id)
+                self.optimized_tts(f"Switched Role to {new_prompt}!")
+                self.play_mp3('output.wav')
+                continue """
+
+            if query.lower() == "voice":
+                rp("Speak your query (or say 'exit' to quit):")
+                self.input_method = self.listen_for_speech()
+                continue
+            
+            if query.lower() == "type":
+                self.input_method = input("Type your question(or type 'exit' to quit): \n")
+                continue
+            
+            if query.lower() == 'exit':
+                rp("Goodbye!")
+                self.optimized_tts("Ok, exiting!")
+                self.play_mp3('output.wav')
+                self.voice_chat_exit = True
+                break
+            
+            result = self.web_search(query)
+            web_context = "\n".join(result) if result else "No Context Available from the websearch!"
+            #vector_context = self.retrieve_context(query)
+            
+            #self.system_prompt = self.system_prompt.replace("<<VSCONTEXT>>", vector_context if vector_context else "No Context Available in the vectorstore!")
+            self.system_prompt = self.system_prompt.replace("<<WSCONTEXT>>", web_context)
+            
+            response = self.bot.chat(query)
+            
+            if "/Store:" in response:
+                url = response.split("/Store:").pop().split(" ")[0]
+                rp(f"Fetching and storing data from link: {url}")
+                try:
+                    self.add_document_from_url(url)
+                except Exception as e:
+                    rp(f"Error while fetching data from {url}! {e}")
+                continue
+            
+            if "/Delete:" in response:
+                document = response.split("/Delete:").pop().split(" ")[0]
+                rp(f"Deleting {document} from vectorstore!")
+                try:
+                    self.delete_document(document)
+                except Exception as e:
+                    rp(f"Error while deleting {document} from vectorstore! {e}")
+
+            rp(f"Chatbot: {response}")
+            
+            self.play_mp3(self.optimized_tts(str(response)))
+
+
+    def initialize_vector_store(
+        self,
+        initial_docs: Union[List[Union[str, Document]], str],
+        embedding_model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+        persist_directory: str = "faiss_index",
+        index_name: str = "document_store"
+    ) -> FAISS:
+        """
+        Initialize a FAISS vector store. If a persistent store exists, load and update it.
+        Otherwise, create a new one from the initial documents.
+
+        Args:
+        initial_docs (Union[List[Union[str, Document]], str]): Initial documents to add if creating a new store.
+        embedding_model_name (str): Name of the HuggingFace embedding model to use.
+        persist_directory (str): Directory to save/load the persistent vector store.
+        index_name (str): Name of the index file.
+
+        Returns:
+        FAISS: The initialized or loaded FAISS vector store.
+        """
+        allow_dangerous_deserialization=True
+        index_file_path = os.path.join(persist_directory, f"{index_name}.faiss")
+        
+        # Convert initial_docs to a list of Document objects
+        if isinstance(initial_docs, str):
+            initial_docs = [Document(page_content=initial_docs)]
+        elif isinstance(initial_docs, list):
+            initial_docs = [
+                doc if isinstance(doc, Document) else Document(page_content=doc)
+                for doc in initial_docs
+            ]
+        
+        if os.path.exists(index_file_path):
+            print(f"Loading existing vector store from {index_file_path}")
+            vector_store = FAISS.load_local(
+                persist_directory, 
+                self.embeddings, 
+                index_name,
+                allow_dangerous_deserialization=allow_dangerous_deserialization
+                )
+            
+            # Update with new documents if any
+            if initial_docs:
+                print(f"Updating vector store with {len(initial_docs)} new documents")
+                vector_store.add_documents(initial_docs)
+                vector_store.save_local(persist_directory, index_name)
+        else:
+            print(f"Creating new vector store with {len(initial_docs)} documents")
+            vector_store = FAISS.from_documents(initial_docs, self.embeddings)
+            
+            # Ensure the directory exists
+            os.makedirs(persist_directory, exist_ok=True)
+            vector_store.save_local(persist_directory, index_name)
+        
+        return vector_store
+    
+    def setup_vector_store(self):   
+        from langchain.docstore import InMemoryDocstore        
+        embedding_size = 384  # Size for all-MiniLM-L6-v2 embeddings
+        index = faiss.IndexFlatL2(embedding_size)
+        docstore = InMemoryDocstore({})
+        
+        self.vector_store = FAISS(
+            self.embeddings, 
+            index, 
+            docstore, 
+            {}
+        )
+
+    """     def setup_vector_store(self):
+            self.vector_store = self.initialize_vector_store(['this your Birth, Rise and Shine a mighty bot']) 
+
+    """
+    def add_documents_folder(self, folder_path):
+        paths=[]
+        for root, _, files in os.walk(folder_path):
+            for file in files:
+                paths.append(os.path.join(root, file))
+        
+        self.add_documents(paths)
+
+    def fetch_document(self, file_path):
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+            return Document(page_content=content)
+            #self.vector_store.add_documents([document])
+    
+    def add_documents(self, documents: List[str]):
+        docs_to_add=[]
+        if not self.vector_store:
+            self.setup_vector_store()
+        for document in documents:
+            docs_to_add.append(self.fetch_document(document))
+
+        self.vector_store.add_documents(docs_to_add)
+        
+        # Print the added documents for verification
+        for i in range(len(docs_to_add)):
+            doc_id = self.vector_store.index_to_docstore_id[i]
+            rp(f"Added document {i}: {self.vector_store.docstore._dict[doc_id]}")
+
+    def add_document_from_url(self, url):
+        if not self.vector_store:
+            self.setup_vector_store()
+        response = requests.get(url)
+        if response.status_code == 200:
+            content = response.text
+            document = Document(page_content=content)
+            self.vector_store.add_documents([document])
+        else:
+            rp(f"Failed to fetch URL content: {response.status_code}")
+
+    def delete_document(self, document):
+        if document in self.vector_store:
+            self.vector_store.delete_document(document)
+            rp(f"Deleted document: {document}")
+        else:
+            rp(f"Document not found: {document}")
+
+    def _add_to_vector_store(self, name, content):
+        document = Document(page_content=content)
+        self.vector_store.add_documents([document])
+        rp(f"Added document to vector store: {name}")
+        # Example of updating the vectorizer (you might need to adjust based on your actual implementation)
+        self.vectorizer.fit_transform(self.compressed_retriever.invoke("*"))
+    
+    def clone_github_repo(self, repo_url, local_path='./repo'):
+        if os.path.exists(local_path):
+            rp("Repository already cloned.") 
+            return local_path
+        Repo.clone_from(repo_url, local_path)
+        return local_path
+    
+    def load_documents(self, repo_url, file_types=['*.py', '*.md', '*.txt', '*.html']):
+        local_repo_path = self.clone_github_repo(repo_url)
+        loader = DirectoryLoader(path=local_repo_path, glob=f"**/{{{','.join(file_types)}}}", show_progress=True, recursive=True)
+        loaded=loader.load()
+        rp(f"Nr. files loaded: {len(loaded)}")
+        return loaded
+    
+    def recursive_glob(self,root_dir, patterns):
+        import fnmatch
+        """Recursively search for files matching the patterns in root_dir.
+
+        Args:
+            root_dir (str): The root directory to start the search from.
+            patterns (list): List of file patterns to search for, e.g., ['*.py', '*.md'].
+
+        Returns:
+            list: List of paths to the files matching the patterns.
+        """
+        matched_files = []
+        for root, dirs, files in os.walk(root_dir):
+            for pattern in patterns:
+                for filename in fnmatch.filter(files, pattern):
+                    matched_files.append(os.path.join(root, filename))
+        return matched_files
+
+
+    def load_documents_from_github(self, repo_url, file_types=['*.py', '*.md', '*.txt', '*.html']):
+        local_repo_path = self.clone_github_repo(repo_url)
+        document_paths = self.recursive_glob(local_repo_path, file_types)
+        rp(f"Found {len(document_paths)} documents")
+        self.add_documents(document_paths)
+        """ loader = DirectoryLoader(path=local_repo_path, glob=f"**/{{{','.join(file_types)}}}", show_progress=True, recursive=True)
+        loaded=loader.load(document_paths)
+        rp(f"Nr. files loaded: {len(loaded)}")
+        return loaded """
+
+    def split_documents(self, documents: list,chunk_s=512,chunk_o=0):
+        split_docs = []
+        for doc in documents:
+            ext = os.path.splitext(getattr(doc, 'source', '') or getattr(doc, 'filename', ''))[1].lower()
+            if ext == '.py':
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.PYTHON, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            elif ext in ['.md', '.markdown']:
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.MARKDOWN, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            elif ext in ['.html', '.htm']:
+                splitter = RecursiveCharacterTextSplitter.from_language(language=Language.HTML, chunk_size=chunk_s, chunk_overlap=chunk_o)
+            else:
+                splitter = CharacterTextSplitter(chunk_size=chunk_s, chunk_overlap=chunk_o, add_start_index=True)
+            
+            split_docs.extend(splitter.split_documents([doc]))
+        return split_docs,splitter
+    
+
+    def save_vectorstore_local(self, folder_path: str="vectorstore", index_name: str = "faiss_index"):
+        """
+        Save the FAISS vectorstore locally with all necessary components.
+
+        Args:
+            folder_path (str): Folder path to save index, docstore, and index_to_docstore_id to.
+            index_name (str): Name for the saved index file (default is "faiss_index").
+        """
+
+        # Get all documents from the vectorstore
+        documents = self.compressed_retriever.invoke("*")<--error
+
+        # Create a new docstore and index_to_docstore_id mapping
+        docstore: Dict[str, Document] = {}
+        index_to_docstore_id: Dict[int, str] = {}
+
+        for i, doc in enumerate(documents):
+            # Generate a unique ID for each document
+            doc_id = str(uuid4())
+            docstore[doc_id] = doc
+            index_to_docstore_id[i] = doc_id
+
+        # Save the FAISS index
+        self.vector_store.save_local(folder_path, index_name)
+
+        # Save the docstore
+        import pickle
+        with open(os.path.join(folder_path, f"{index_name}_docstore.pkl"), "wb") as f:
+            pickle.dump(docstore, f)
+
+        # Save the index_to_docstore_id mapping
+        with open(os.path.join(folder_path, f"{index_name}_index_to_docstore_id.pkl"), "wb") as f:
+            pickle.dump(index_to_docstore_id, f)
+
+        rp(f"Vectorstore saved successfully to {folder_path}")
+        return folder_path
+    
+
+    @classmethod
+    def load_vectorstore_local(cls, folder_path: str, index_name: str = "faiss_index", embeddings=None):
+        """
+        Load a previously saved FAISS vectorstore.
+        Args:
+            folder_path (str): Folder path where the index, docstore, and index_to_docstore_id are saved.
+            index_name (str): Name of the saved index file (default is "faiss_index").
+            embeddings: The embeddings object to use (must be the same type used when saving).
+        Returns:
+            FAISS: Loaded FAISS vectorstore
+        """
+        # Ensure you trust the source of the pickle file before setting this to True
+        allow_dangerous_deserialization = True
+
+        # Load the docstore
+        with open(os.path.join(folder_path, f"{index_name}_docstore.pkl"), "rb") as f:
+            docstore = pickle.load(f)
+        # Load the index_to_docstore_id mapping
+        with open(os.path.join(folder_path, f"{index_name}_index_to_docstore_id.pkl"), "rb") as f:
+            index_to_docstore_id = pickle.load(f)
+
+        # Load the FAISS index
+        vectorstore = FAISS.load_local(
+            folder_path, 
+            embeddings, 
+            index_name,
+            allow_dangerous_deserialization=allow_dangerous_deserialization
+            )
+        # Reconstruct the FAISS object with the loaded components
+        vectorstore.docstore = docstore
+        vectorstore.index_to_docstore_id = index_to_docstore_id
+
+        return vectorstore
+    
+    def create_vectorstore_from_github(self):
+        documents = self.load_documents_from_github(self.repo_url)
+        split_docs,splitter = self.split_documents(documents,512,0)
+        self.latest_splitter=splitter
+        self.vector_store = FAISS.from_documents(split_docs, self.embeddings)
+        self.vector_store.save_local()
+        rp(f"Vectorstore created with {len(split_docs)} documents.")
+
+    def update_vectorstore(self, new_documents):
+        split_docs,splitter = self.split_documents(new_documents)
+        self.latest_splitter=splitter
+        self.vector_store.add_documents(split_docs)
+        rp(f"Vectorstore updated with {len(split_docs)} new documents.")
+    
+
+    def retrieve_with_chain(self, query, mode='high'):
+        if mode == 'high':
+            return self.compressed_retriever.invoke({"input": query})
+        else:
+            return self.retriever.invoke({"input": query})
+
+    def generate_code(self, prompt):
+        self.system_prompt=self.prompts["code_generator_prompt"]
+        return self.send_message(prompt)
+
+    def debug_script(self, script):
+        self.system_prompt = self.prompts["script_debugger_prompt"]
+        return self.send_message(f"Debug the following script:\n\n{script}")
+
+    def test_software(self, software_description):
+        self.system_prompt = self.prompts["software_tester_prompt"]
+        return self.send_message(f"Create a test plan for the following software:\n\n{software_description}")
+
+    def parse_todo(self, todo_list):
+        self.system_prompt = self.prompts["todo_parser_prompt"]
+        return self.send_message(f"Parse and organize the following TODO list:\n\n{todo_list}")
+
+    def tell_story(self, prompt):
+        self.system_prompt = self.prompts["story_teller_prompt"]
+        return self.stream_response(f"Tell a story based on this prompt:\n\n{prompt}")
+
+    def act_as_copilot(self, task):
+        self.system_prompt = self.prompts["copilot_prompt"]
+        return self.send_message(f"Assist me as a copilot for the following task:\n\n{task}")
+
+    def control_iterations(self, task, max_iterations=5):
+        self.system_prompt = self.prompts["iteration_controller_prompt"]
+        iteration = 0
+        result = ""
+        while iteration < max_iterations:
+            response = self.send_message(f"Iteration {iteration + 1} for task:\n\n{task}\n\nCurrent result:\n{result}")
+            result += f"\nIteration {iteration + 1}:\n{response}"
+            if "TASK_COMPLETE" in response:
+                break
+            iteration += 1
+        return result
+
+    def voice_command_mode(self):
+        rp("Entering voice command mode. Speak your commands.")
+        while True:
+            command = self.listen_for_speech()
+            if command is None:
+                continue
+            if command.lower() == "exit voice mode":
+                rp("Exiting voice command mode.")
+                break
+            response = self.process_voice_command(command)
+            rp(f"Assistant: {response}")
+            self.optimized_tts(response)
+            self.play_mp3('output.wav')
+
+    def process_voice_command(self, command):
+        if "generate code" in command.lower():
+            return self.generate_code(command)
+        elif "debug script" in command.lower():
+            return self.debug_script(command)
+        elif "test software" in command.lower():
+            return self.test_software(command)
+        elif "parse todo" in command.lower():
+            return self.parse_todo(command)
+        elif "tell story" in command.lower():
+            return self.tell_story(command)
+        elif "act as copilot" in command.lower():
+            return self.act_as_copilot(command)
+        else:
+            return self.send_message(command)
+
+    def interactive_mode(self):
+        rp("Entering interactive mode. Type 'exit' to quit, 'voice' for voice input, or 'command' for specific functions.")
+        while True:
+            user_input = input("You: ")
+            if user_input.lower() == 'exit':
+                rp("Exiting interactive mode.")
+                break
+            elif user_input.lower() == 'voice':
+                self.voice_command_mode()
+            elif user_input.lower() == 'command':
+                self.command_mode()
+            else:
+                response = self.send_message(user_input)
+                rp(f"Assistant: {response}")
+
+    def command_mode(self):
+        rp("Entering command mode. Available commands: generate_code, debug_script, test_software, parse_todo, tell_story, copilot, iterate")
+        while True:
+            command = input("Enter command (or 'exit' to return to interactive mode): ")
+            if command.lower() == 'exit':
+                rp("Exiting command mode.")
+                break
+            self.execute_command(command)
+
+    def execute_command(self, command):
+        if command == "add_to_vectorstore":
+            prompt = input("Enter list of files, folders, urls or repos with knowledge to add:")
+            response = self.generate_code(prompt)
+        if command == "generate_code":
+            file_name = input("Enter script filename:")
+            prompt = input("Enter code generation prompt:")
+            response = self.generate_code(prompt)
+        elif command == "debug_script":
+            script = input("Enter script to debug:")
+            response = self.debug_script(script)
+        elif command == "test_script":
+            description = input("Enter path to script:")
+            response = self.test_software(description)
+        elif command == "parse_todo":
+            todo_list = input("Enter TODO list:")
+            response = self.parse_todo(todo_list)
+        elif command == "tell_story":
+            prompt = input("Enter story prompt:")
+            response = self.tell_story(prompt)
+        elif command == "copilot":
+            task = input("Enter task for copilot:")
+            response = self.act_as_copilot(task)
+        elif command == "iterate":
+            task = input("Enter task for iteration:")
+            max_iterations = int(input("Enter maximum number of iterations: "))
+            response = self.control_iterations(task, max_iterations)
+        else:
+            response = "Unknown command. Please try again."
+        
+        rp(f"Assistant: {response}")
+
+    def run(self):
+        rp("Welcome to the Advanced AI Toolkit!")
+        rp("Choose a mode to start:")
+        rp("1. Interactive Chat")
+        rp("2. Voice Chat")
+        rp("3. Command Mode")
+        choice = input("Enter your choice (1/2/3): ")
+        
+        if choice == '1':
+            self.interactive_mode()
+        elif choice == '2':
+            self.continuous_voice_chat()
+        elif choice == '3':
+            self.command_mode()
+        else:
+            rp("Invalid choice. Exiting.")
+
+if __name__ == "__main__":
+    email = os.getenv("EMAIL")
+    password = os.getenv("PASSWD")
+    toolkit = UberToolkit(email, password)
+    toolkit.run()
+
+