Update app.py

app.py

@@ -1,25 +1,28 @@
-import gradio as gr
+from adapters import AutoAdapterModel  # Ensure this library is correctly installed
 from transformers import AutoTokenizer
+import gradio as gr
 import onnxruntime as ort
 import numpy as np
 import string
 from huggingface_hub import InferenceClient
 import os
 
-# Initialize Qwen client
-qwen_client = InferenceClient(os.environ.get('MODEL_ID'))
+# Load Base Model and Adapter
+BASE_MODEL = "Qwen/Qwen2.5-1.5B-Instruct"  # Replace with the actual base model ID
+ADAPTER_NAME = "ystemsrx/Qwen2.5-Sex"  # Replace with the correct adapter name
 
-# Model and ONNX setup
-HG_MODEL = "livekit/turn-detector"
+model = AutoAdapterModel.from_pretrained(BASE_MODEL)
+model.load_adapter(ADAPTER_NAME, set_active=True)
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+
+# ONNX setup
 ONNX_FILENAME = "model_quantized.onnx"
+onnx_session = ort.InferenceSession(ONNX_FILENAME, providers=["CPUExecutionProvider"])
+
 PUNCS = string.punctuation.replace("'", "")
-MAX_HISTORY = 4  # Adjusted to use the last 4 messages
+MAX_HISTORY = 4
 MAX_HISTORY_TOKENS = 512
-EOU_THRESHOLD = 0.5  # Updated threshold to match original
-
-# Initialize ONNX model
-tokenizer = AutoTokenizer.from_pretrained(HG_MODEL)
-onnx_session = ort.InferenceSession(ONNX_FILENAME, providers=["CPUExecutionProvider"])
+EOU_THRESHOLD = 0.5
 
 # Softmax function
 def softmax(logits):
@@ -42,18 +45,15 @@ def format_chat_ctx(chat_ctx):
                 msg["content"] = content
                 new_chat_ctx.append(msg)
 
-    # Tokenize with chat template
     convo_text = tokenizer.apply_chat_template(
         new_chat_ctx, add_generation_prompt=False, add_special_tokens=False, tokenize=False
     )
-
-    # Remove EOU token from the current utterance
     ix = convo_text.rfind("<|im_end|>")
     return convo_text[:ix]
 
 # Calculate EOU probability
 def calculate_eou(chat_ctx, session):
-    formatted_text = format_chat_ctx(chat_ctx[-MAX_HISTORY:])  # Use the last 4 messages
+    formatted_text = format_chat_ctx(chat_ctx[-MAX_HISTORY:])
     inputs = tokenizer(
         formatted_text,
         return_tensors="np",
@@ -67,7 +67,6 @@ def calculate_eou(chat_ctx, session):
     eou_token_id = tokenizer.encode("<|im_end|>")[-1]
     return probs[eou_token_id]
 
-
 # Respond function
 def respond(
     message,
@@ -76,28 +75,18 @@ def respond(
     temperature,
     top_p,
 ):
-    # Keep the last 4 conversation pairs (user-assistant)
     messages = [{"role": "system", "content": os.environ.get("CHARACTER_DESC")}]
-
-    for val in history[-10:]:  # Only use the last 4 pairs
+    for val in history[-10:]:
         if val[0]:
             messages.append({"role": "user", "content": val[0]})
         if val[1]:
             messages.append({"role": "assistant", "content": val[1]})
-
-    # Add the new user message to the context
     messages.append({"role": "user", "content": message})
-
-    # Calculate EOU probability
     eou_prob = calculate_eou(messages, onnx_session)
-    print(f"EOU Probability: {eou_prob}")  # Debug output
-
-    # If EOU is below the threshold, ask for more input
+    print(f"EOU Probability: {eou_prob}")
     if eou_prob < EOU_THRESHOLD:
         yield "[Waiting for user to continue input...]"
         return
-
-    # Generate response with Qwen
     response = ""
     for message in qwen_client.chat_completion(
         messages,
@@ -109,23 +98,11 @@ def respond(
         token = message.choices[0].delta.content
         response += token
         yield response
-    
     print(f"Generated response: {response}")
 
-
 # Gradio interface
 demo = gr.ChatInterface(
     respond,
-    # additional_inputs=[
-    #     # Commented out to disable user modification of the system message
-    #     # gr.Textbox(value="You are an assistant.", label="System message"),
-    #     gr.Slider(minimum=1, maximum=4096, value=256, step=1, label="Max new tokens"),
-    #     gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-    #     gr.Slider(
-    #         minimum=0.1,
-    #         maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"
-    #     ),
-    # ],
 )
 
 if __name__ == "__main__":