Upload 5 files

app.py

@@ -1,63 +1,262 @@
+import os
 import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
-
-
-if __name__ == "__main__":
-    demo.launch()
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import pandas as pd
+from datetime import datetime, timedelta, timezone
+import torch
+from config import hugging_face_token, init_google_sheets_client, models, quantized_models, default_model_name, user_names, google_sheets_name, MAX_INTERACTIONS
+import spaces
+
+# Hack for ZeroGPU
+torch.jit.script = lambda f: f
+
+# Initialize Google Sheets client
+client = init_google_sheets_client()
+sheet = client.open(google_sheets_name)
+stories_sheet = sheet.worksheet("Stories")
+system_prompts_sheet = sheet.worksheet("System Prompts")
+
+# Load stories from Google Sheets
+def load_stories():
+    stories_data = stories_sheet.get_all_values()
+    stories = [{"title": story[0], "story": story[1]} for story in stories_data if story[0] != "Title"]  # Skip header row
+    return stories
+
+# Load system prompts from Google Sheets
+def load_system_prompts():
+    system_prompts_data = system_prompts_sheet.get_all_values()
+    system_prompts = [prompt[0] for prompt in system_prompts_data[1:]]  # Skip header row
+    return system_prompts
+
+# Load available stories and system prompts
+stories = load_stories()
+system_prompts = load_system_prompts()
+
+# Initialize the selected model
+selected_model = default_model_name
+tokenizer, model = None, None
+
+# Initialize the data list
+data = []
+
+# Load the model and tokenizer once at the beginning
+def load_model(model_name):
+    global tokenizer, model, selected_model
+    try:
+        # Release the memory of the previous model if exists
+        if model is not None:
+            del model
+            torch.cuda.empty_cache()
+        
+        # Check if the model is in models or quantized_models and load accordingly
+        if model_name in models:
+            model_path = models[model_name]
+        elif model_name in quantized_models:
+            model_path = quantized_models[model_name]
+        else:
+            raise ValueError(f"Model {model_name} not found in either models or quantized_models.")
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_path, 
+            padding_side='left', 
+            token=hugging_face_token, 
+            trust_remote_code=True
+        )
+        
+        # Ensure the padding token is set
+        if tokenizer.pad_token is None:
+            tokenizer.pad_token = tokenizer.eos_token
+            tokenizer.add_special_tokens({'pad_token': tokenizer.eos_token})
+        
+        model = AutoModelForCausalLM.from_pretrained(
+            model_path, 
+            token=hugging_face_token, 
+            trust_remote_code=True
+        )
+        
+        # Only move to CUDA if it's not a quantized model
+        if model_name not in quantized_models:
+            model = model.to("cuda")
+        
+        selected_model = model_name
+    except Exception as e:
+        print(f"Error loading model {model_name}: {e}")
+        raise e
+    return tokenizer, model
+
+
+
+# Ensure the initial model is loaded
+tokenizer, model = load_model(selected_model)
+
+# Chat history
+chat_history = []
+
+# Function to handle interaction with model
+@spaces.GPU
+def interact(user_input, history, interaction_count, model_name):
+    global tokenizer, model
+    try:
+        if tokenizer is None or model is None:
+            raise ValueError("Tokenizer or model is not initialized.")
+        
+        # Determine the device to use (either CUDA if available, or CPU)
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        
+        # Only move the model to the device if it's not a quantized model
+        if model_name not in quantized_models:
+            model = model.to(device)
+        
+        if interaction_count >= MAX_INTERACTIONS:
+            user_input += ". Thank you for your questions. Our session is now over. Goodbye!"
+        
+        messages = history + [{"role": "user", "content": user_input}]
+        
+        # Ensure roles alternate correctly
+        for i in range(1, len(messages)):
+            if messages[i-1].get("role") == messages[i].get("role"):
+                raise ValueError("Conversation roles must alternate user/assistant/user/assistant/...")
+        
+        prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        
+        # Move input tensor to the correct device
+        input_ids = tokenizer(prompt, return_tensors='pt').input_ids.to(device)
+        chat_history_ids = model.generate(input_ids, max_new_tokens=100, pad_token_id=tokenizer.eos_token_id, temperature=0.1)
+        response = tokenizer.decode(chat_history_ids[:, input_ids.shape[-1]:][0], skip_special_tokens=True)
+        
+        # Update chat history with generated response
+        history.append({"role": "user", "content": user_input})
+        history.append({"role": "assistant", "content": response})
+        
+        interaction_count += 1
+        
+        formatted_history = [(entry["content"], None) if entry["role"] == "user" else (None, entry["content"]) for entry in history if entry["role"] in ["user", "assistant"]]
+        
+        return "", formatted_history, history, interaction_count
+    except Exception as e:
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        print(f"Error during interaction: {e}")
+        raise gr.Error(f"An error occurred during interaction: {str(e)}")
+
+
+# Function to send selected story and initial message
+def send_selected_story(title, model_name, system_prompt):
+    global chat_history
+    global selected_story
+    global data  # Ensure data is reset
+    data = []  # Reset data for new story
+    interaction_count = 1  # Reset interaction count for new story
+    tokenizer, model = load_model(model_name)  # Load the appropriate model
+    selected_story = title
+    for story in stories:
+        if story["title"] == title:
+            system_prompt = f"""
+{system_prompt}
+Here is the story:
+---
+{story['story']}
+---
+            """
+            combined_message = system_prompt.strip()
+            if combined_message:
+                chat_history = []  # Reset chat history
+                chat_history.append({"role": "system", "content": combined_message})
+                question_prompt = "Please ask a simple question about the story to encourage interaction."
+                _, formatted_history, chat_history, interaction_count = interact(question_prompt, chat_history, interaction_count, model_name)
+
+                return formatted_history, chat_history, gr.update(value=[]), story["story"]
+            else:
+                print("Combined message is empty.")
+        else:
+            print("Story title does not match.")
+
+
+# Function to save comment and score
+def save_comment_score(chat_responses, score, comment, story_name, user_name, system_prompt):
+    full_chat_history = ""
+
+    # Create formatted chat history with roles
+    for message in chat_responses:
+        if message[0]:  # User message
+            full_chat_history += f"User: {message[0]}\n"
+        if message[1]:  # Assistant message
+            full_chat_history += f"Assistant: {message[1]}\n"
+
+    timestamp = datetime.now(timezone.utc) - timedelta(hours=3)  # Adjust to GMT-3
+    timestamp_str = timestamp.strftime("%Y-%m-%d %H:%M:%S")
+    model_name = selected_model
+
+    # Append data to local data storage
+    data.append([
+        timestamp_str,
+        user_name,
+        model_name,
+        system_prompt,
+        story_name,
+        full_chat_history,
+        score,
+        comment
+    ])
+
+    # Append data to Google Sheets
+    try:
+        user_sheet = client.open(google_sheets_name).worksheet(user_name)
+    except gspread.exceptions.WorksheetNotFound:
+        user_sheet = client.open(google_sheets_name).add_worksheet(title=user_name, rows="100", cols="20")
+    
+    user_sheet.append_row([timestamp_str, user_name, model_name, system_prompt, story_name, full_chat_history, score, comment])
+
+    df = pd.DataFrame(data, columns=["Timestamp", "User Name", "Model Name", "System Prompt", "Story Name", "Chat History", "Score", "Comment"])
+    return df[["Chat History", "Score", "Comment"]], gr.update(value="")  # Show only the required columns and clear the comment input box
+
+# Function to load user guide from a file
+def load_user_guide():
+    with open('user_guide.txt', 'r') as file:
+        return file.read()
+
+# Combine both model dictionaries
+all_models = {**models, **quantized_models}
+
+# Create the chat interface using Gradio Blocks
+with gr.Blocks() as demo:
+    with gr.Tabs():
+        with gr.TabItem("Chat"):
+            gr.Markdown("# Demo Chatbot V3")
+
+            gr.Markdown("## Context")
+            with gr.Group():
+                model_dropdown = gr.Dropdown(choices=list(all_models.keys()), label="Select Model", value=default_model_name)
+                user_dropdown = gr.Dropdown(choices=user_names, label="Select User Name")
+                initial_story = stories[0]["title"] if stories else None
+                story_dropdown = gr.Dropdown(choices=[story["title"] for story in stories], label="Select Story", value=initial_story)
+                system_prompt_dropdown = gr.Dropdown(choices=system_prompts, label="Select System Prompt", value=system_prompts[0])
+                send_story_button = gr.Button("Send Story")
+
+            gr.Markdown("## Chat")
+            with gr.Group():
+                selected_story_textbox = gr.Textbox(label="Selected Story", lines=10, interactive=False)
+                chatbot_output = gr.Chatbot(label="Chat History")
+                chatbot_input = gr.Textbox(placeholder="Type your message here...", label="User Input")
+                send_message_button = gr.Button("Send")
+
+            gr.Markdown("## Evaluation")
+            with gr.Group():
+                score_input = gr.Slider(minimum=0, maximum=5, step=1, label="Score")
+                comment_input = gr.Textbox(placeholder="Add a comment...", label="Comment")
+                save_button = gr.Button("Save Score and Comment")
+                data_table = gr.DataFrame(headers=["Chat History", "Score", "Comment"])
+
+        with gr.TabItem("User Guide"):
+            gr.Textbox(label="User Guide", value=load_user_guide(), lines=20)
+            
+    chat_history_json = gr.JSON(value=[], visible=False)
+    interaction_count = gr.Number(value=0, visible=False)
+
+
+
+    send_story_button.click(fn=send_selected_story, inputs=[story_dropdown, model_dropdown, system_prompt_dropdown], outputs=[chatbot_output, chat_history_json, data_table, selected_story_textbox])
+    send_message_button.click(fn=interact, inputs=[chatbot_input, chat_history_json, interaction_count, model_dropdown], outputs=[chatbot_input, chatbot_output, chat_history_json, interaction_count])
+    save_button.click(fn=save_comment_score, inputs=[chatbot_output, score_input, comment_input, story_dropdown, user_dropdown, system_prompt_dropdown], outputs=[data_table, comment_input])
+
+demo.launch()

config.py

@@ -0,0 +1,35 @@
+import os
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+
+# Read the authentication token from the environment variable
+hugging_face_token = os.getenv("HUGGING_FACE_TOKEN")
+
+# Google Sheets configuration
+def init_google_sheets_client():
+    scope = ["https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive"]
+    creds = ServiceAccountCredentials.from_json_keyfile_name('tokyo-portal-326513-90aee094bab9.json', scope)
+    return gspread.authorize(creds)
+
+# Google Sheets name
+google_sheets_name = "Chatbot Test"
+
+# Define available models
+models = {
+    "Meta-Llama-3-8B-Instruct": "meta-llama/Meta-Llama-3-8B-Instruct",
+    "Llama-2-7B-Chat": "meta-llama/Llama-2-7b-chat-hf",
+    "Yi-6B-Chat": "01-ai/Yi-6B-Chat",
+    "Qwen2-7B-Instruct": "Qwen/Qwen2-7B-Instruct"
+}
+# List of models fine-tuned in 4-bit or 8-bit
+quantized_models = {
+    "Llama-3-8B-Finetuning-Stories": "rodrisouza/Llama-3-8B-Finetuning-Stories",
+}
+
+# Default model name
+default_model_name = "Meta-Llama-3-8B-Instruct"
+
+# Define available user names
+user_names = ["Laura Musto", "Brian Carpenter", "Germán Capdehourat", "Isabel Amigo", "Aiala Rosá", "Luis Chiruzzo", "Igancio Sastre", "Santiago Góngora", "Ignacio Remersaro", "Rodrigo Souza"]
+
+MAX_INTERACTIONS = 5 

requirements.txt

@@ -1 +1,9 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+minijinja
+transformers
+torch
+pandas
+gspread
+oauth2client
+accelerate
+bitsandbytes

tokyo-portal-326513-90aee094bab9 (1).json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "tokyo-portal-326513",
+  "private_key_id": "90aee094bab930aedc4ad7bdb17626ef345ef57a",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDA9rv7k4RnEi9G\nngtf7+cyv7ReziihB22cz3oWBypfjLw9kwoKTRiBQJyltSWhTUaC+ZBQdLcVQ0Xh\nKgw8PXZ25Q1xkrvLYbR5TD4Id1yfGk0s9ymhLa/OqsUmpbaleRsLcfxETpz6dfIN\n4t7md1S5OusXMirWm7PXjBVpU+fp2eCkJtrSFEWqdZo4LkJ3s6oOQzI5GWac0ntX\ntYcdxfl9Ul/XqE23s65rXB/rHxx9+NMwgnT4QZRXOuYhJqLYP5TPpNsz7MTRf9pt\n7ZCkIJQpwwAcvDW0onenOCCYMn11ntDJICiJsdyaoREfodx2D4dhNyxw2dT73wtm\nJsLETzNbAgMBAAECggEAAZyuElp3ozLMMU9Vsf37lPl0BMy9GydIO6SiFPgHWxf+\nZJdLYK2nNEE5bcr3HwNZwPNbyMhI1UUkGs7ynjg3OqQMOjQrHHZa/x5rcD+rBus9\n1M/VybJrgxL7JakBvx0lZDGWFW4Asrj0hjgse4lorEl5OE7je7p+RKE3dpNMXzA+\nkPe7y5RG74cwIEg2C7y/WOQxuDEELqyse7HOXR6+BSVntb6eVhrt4xTn3KoGWxhE\nWd9maD3rMsSkmQtk45wWSvwdj6Pkfp/DL5DX2iIprFIpbveYaa907W6BD8Kz4xnA\nFS5EDdqCxqURJk9p/ADxjSPxnbmqVPrUK3KsHWkyIQKBgQDgeUgEY4/bJKtx12Sl\nw/acvGA6OxHNVNN4R8YLC1xDUg+csy09vXTZ6VcHqae417k+KXe3beBoE2k7mX5b\nZNShtIBkm/g0Ei4t80mgEaatLBddZghoiLJvHOCdpTmMfNt41KWzyr139PgT/KSK\n5iQnTsbxPUmjgS0h58KfM6CfOwKBgQDcEIu4IGXIXQ1qydXZgvkYte3nrjJw+ump\n0OklIvZjOcptgWAjguNr/6l5tqXnc3VhD5Mkk1hRHckUHYU93RoalTk/xGWoHlnx\n/rfZq/PMHNd4GoRl8GHWGAzviaS1Nqcz+8vwXW8Sg5mnoIFFrWkg6NN418k2FlLs\nZ1Hfuia6YQKBgDt+qaHQKZ6xl2cy6ZAt4j0kiHSml09OvkXQ8CKZPjjxlQW5T7Jf\ndosTF2gGhVeuhYB+SSaqzqGHE4siYaUjkl/RkX4ZAK38a5TuOINeakjXuNGea3gZ\nUwG0K6xc0GX16c2avthqUdFrch0KBypxWP+6F2x1DAF0CVxFKwY7DsVlAoGBAIOa\nAXQjK7seqp9qBFEHWh/E7HjYW6Hk60d6f3KN7fp43O2PqQaTh51WTutSpvQ6v79D\nqSL+WppnzZAR91R/nNS42Huh34kiXXeydA/gHPO7a7+PXA36Kwf1agb1sWINRQB5\nXARqW7oeqQztl2Eryuq7UXu6z8FgqEZnozbAA9kBAoGBAI2cycVnPOHfRo24KLsI\nOIhfkgUn2VPgGpBig1QVaGdi0fLsKPX7HnHbui9m5dXwgps/w8sUd/hwoCu0zR/C\nedYc+G1VmUikJ2fLXj3OX1axS4oGTg3ZuN+O4/GsA65xrTotIJBxp9RQz+HXTzlU\nBGg3nYWf2DB1d3oZ8sFU4T3S\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "108995062104802726229",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/chatbot-test%40tokyo-portal-326513.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

user_guide.txt

@@ -0,0 +1,24 @@
+La aplicación está organizada en tres bloques principales: Contexto, Chat y Evaluación.
+
+1. Contexto
+En este bloque, configuramos el contexto antes de interactuar con el chat.
+1. Seleccionar el modelo: Elige el modelo que deseas probar de la lista de modelos disponibles.
+2. Seleccionar el usuario: Elige el usuario que está realizando las pruebas. Es importante seleccionar el usuario correcto para guardar los resultados adecuadamente.
+3. Seleccionar el cuento: Elige el cuento que se trabajará con el modelo.
+4. Seleccionar el System Prompt: Configura el mensaje de sistema que guiará la interacción con el modelo.
+5. Enviar el cuento: Haz clic en "Send Story" para enviar el cuento seleccionado y configurar el contexto.
+
+2. Chat
+En este bloque, interactuamos con el modelo de lenguaje.
+1. Visualizar el cuento seleccionado: En el campo “Selected Story” se muestra el cuento enviado anteriormente en el contexto.
+2. Escribir el mensaje: Introduce tu mensaje en el campo “User Input”.
+3. Enviar el mensaje: Haz clic en "Send" para enviar tu mensaje al modelo y recibir una respuesta.
+
+3. Evaluación
+En este bloque, evaluamos la última interacción realizada entre el usuario y el chat.
+1. Asignar un puntaje: Selecciona el puntaje que deseas asignar al resultado de la interacción.
+2. Escribir un comentario: Introduce un comentario sobre el resultado de la interacción.
+3. Guardar la evaluación: Haz clic en "Save Score and Comment" para guardar el puntaje y el comentario.
+
+Obs: La lista de cuentos y system prompts disponibles, así como los resultados, se encuentran en el siguiente link.
+https://docs.google.com/spreadsheets/d/1EY0vYkzsrSYW5PNWDDTUHlL4ahTym3Wj1etxRHQY3c8/edit?usp=sharing