Update app.py

app.py

@@ -23,16 +23,19 @@ llm = None
 llm_model = None
 
 @spaces.GPU(duration=120)
-def translate_text(
-    input_text: str,
-    model: str,
-    system_message: str,
-    max_tokens: int,
-    temperature: float,
-    top_p: float,
-    top_k: int,
-    repeat_penalty: float
-) -> str:
+def respond(
+    message,
+    history: list[tuple[str, str]],
+    model,
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+    top_k,
+    repeat_penalty,
+):
+    chat_template = MessagesFormatterType.GEMMA_2
+
     global llm
     global llm_model
     
@@ -51,45 +54,50 @@ def translate_text(
     agent = LlamaCppAgent(
         provider,
         system_prompt=f"{system_message}",
-        predefined_messages_formatter_type=MessagesFormatterType.GEMMA_2,
+        predefined_messages_formatter_type=chat_template,
         debug_output=True
     )
     
     settings = provider.get_provider_default_settings()
-    settings.temperature = temperature
+    settings.temperature = 0.1
     settings.top_k = top_k
-    settings.top_p = top_p
+    settings.top_p = 0.3
     settings.max_tokens = max_tokens
-    settings.repeat_penalty = repeat_penalty
-    settings.stream = False
+    settings.repeat_penalty = 1.0
+    settings.stream = True
 
     messages = BasicChatHistory()
-    user_message = {
-        'role': Roles.user,
-        'content': input_text
-    }
-    messages.add_message(user_message)
+
+    for msn in history:
+        user = {
+            'role': Roles.user,
+            'content': msn[0]
+        }
+        assistant = {
+            'role': Roles.assistant,
+            'content': msn[1]
+        }
+        messages.add_message(user)
+        messages.add_message(assistant)
     
-    response = agent.get_chat_response(
-        input_text,
+    stream = agent.get_chat_response(
+        message,
         llm_sampling_settings=settings,
         chat_history=messages,
-        returns_streaming_generator=False,
+        returns_streaming_generator=True,
         print_output=False
     )
     
-    return response
+    outputs = ""
+    for output in stream:
+        outputs += output
+        yield outputs
 
-description = """<p align="center">Enter Japanese text to translate it into Simplified Chinese.</p>"""
+description = """<p align="center">Defaults to Sakura-14B-Qwen2beta (you can switch from additional inputs)</p>"""
 
-demo = gr.Interface(
-    fn=translate_text,
-    inputs=[
-        gr.Textbox(
-            lines=2,
-            placeholder="請輸入日文",
-            label="Input Japanese Text"
-        ),
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
         gr.Dropdown([
                 'sakura-14b-qwen2beta-v0.9.2-q6k.gguf'
             ],
@@ -97,7 +105,7 @@ demo = gr.Interface(
             label="Model"
         ),
         gr.Textbox(value="你是一个轻小说翻译模型，可以流畅通顺地以日本轻小说的风格将日文翻译成简体中文，并联系上下文正确使用人称代词，不擅自添加原文中没有的代词。", label="System message"),
-        gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max tokens"),
+        gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.1, step=0.1, label="Temperature"),
         gr.Slider(
             minimum=0.1,
@@ -109,7 +117,7 @@ demo = gr.Interface(
         gr.Slider(
             minimum=0,
             maximum=100,
-            value=1,
+            value=40,
             step=1,
             label="Top-k",
         ),
@@ -121,13 +129,18 @@ demo = gr.Interface(
             label="Repetition penalty",
         ),
     ],
-    outputs=gr.Textbox(
-        lines=2,
-        label="Translated Chinese Text"
-    ),
-    title="Japanese to Chinese Translator",
-    description=description
+    retry_btn="Retry",
+    undo_btn="Undo",
+    clear_btn="Clear",
+    submit_btn="Send",
+    title="Chat with Sakura-14B-Qwen2beta using llama.cpp", 
+    description=description,
+    chatbot=gr.Chatbot(
+        scale=1, 
+        likeable=False,
+        show_copy_button=True
+    )
 )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()