Update app.py

app.py

@@ -67,67 +67,54 @@ def calculate_eou(chat_ctx, session):
     eou_token_id = tokenizer.encode("<|im_end|>")[-1]
     return probs[eou_token_id]
 
-
-# Respond function
+# Respond function with default parameters
 def respond(
     message,
     history: list[tuple[str, str]],
-    max_tokens,
-    temperature,
-    top_p,
+    max_tokens=256,
+    temperature=0.7,
+    top_p=0.95,
 ):
-    # Keep the last 4 conversation pairs (user-assistant)
-    messages = [{"role": "system", "content": os.environ.get("CHARACTER_DESC")}]
+    messages = [{"role": "system", "content": os.environ.get("CHARACTER_DESC", "You are a helpful assistant.")}]
 
-    for val in history[-10:]:  # Only use the last 4 pairs
+    for val in history[-MAX_HISTORY:]:  # Use last 4 pairs
         if val[0]:
             messages.append({"role": "user", "content": val[0]})
         if val[1]:
             messages.append({"role": "assistant", "content": val[1]})
 
-    # Add the new user message to the context
     messages.append({"role": "user", "content": message})
 
-    # Calculate EOU probability
     eou_prob = calculate_eou(messages, onnx_session)
-    print(f"EOU Probability: {eou_prob}")  # Debug output
+    print(f"EOU Probability: {eou_prob}")
 
-    # If EOU is below the threshold, ask for more input
     if eou_prob < EOU_THRESHOLD:
         yield "[Waiting for user to continue input...]"
         return
 
-    # Generate response with Qwen
     response = ""
-    for message in qwen_client.chat_completion(
+    for chunk in qwen_client.chat_completion(
         messages,
         max_tokens=max_tokens,
         stream=True,
         temperature=temperature,
         top_p=top_p,
     ):
-        token = message.choices[0].delta.content
+        token = chunk.choices[0].delta.content or ""
         response += token
         yield response
-    
-    print(f"Generated response: {response}")
 
+    print(f"Generated response: {response}")
 
-# Gradio interface
+# Gradio interface with additional inputs
 demo = gr.ChatInterface(
     respond,
-    # additional_inputs=[
-    #     # Commented out to disable user modification of the system message
-    #     # gr.Textbox(value="You are an assistant.", label="System message"),
-    #     gr.Slider(minimum=1, maximum=4096, value=256, step=1, label="Max new tokens"),
-    #     gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-    #     gr.Slider(
-    #         minimum=0.1,
-    #         maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"
-    #     ),
-    # ],
+    additional_inputs=[
+        gr.Slider(minimum=1, maximum=4096, value=256, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
+    ],
 )
 
 if __name__ == "__main__":
-    demo.launch()
-
+    demo.launch()