Update app.py

app.py

@@ -10,8 +10,8 @@ import sentencepiece
 model = AutoModelForCausalLM.from_pretrained("01-ai/Yi-34B-200K", device_map="auto", torch_dtype="auto", trust_remote_code=True)
 tokenizer = YiTokenizer(vocab_file="./tokenizer.model")
 
-def run(message, chat_history, system_prompt, max_new_tokens=1024, temperature=0.3, top_p=0.9, top_k=50):
-    prompt = get_prompt(message, chat_history, system_prompt)
+def run(message, chat_history, max_new_tokens=100000, temperature=3.5, top_p=0.9, top_k=800):
+    prompt = get_prompt(message, chat_history)
 
     # Encode the prompt to tensor
     input_ids = tokenizer.encode(prompt, return_tensors='pt')
@@ -32,16 +32,16 @@ def run(message, chat_history, system_prompt, max_new_tokens=1024, temperature=0
     response = tokenizer.decode(response_ids[:, input_ids.shape[-1]:][0], skip_special_tokens=True)
     return response
 
-def get_prompt(message, chat_history, system_prompt):
-    texts = [f"<s>[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n"]
+def get_prompt(message, chat_history):
+    texts = []
 
     do_strip = False
     for user_input, response in chat_history:
         user_input = user_input.strip() if do_strip else user_input
         do_strip = True
-        texts.append(f"{user_input} [/INST] {response.strip()} </s><s>[INST] ")
+        texts.append(f" {response.strip()} {user_input} ")
     message = message.strip() if do_strip else message
-    texts.append(f"{message} [/INST]")
+    texts.append(f"{message}")
     return ''.join(texts)
 
 DESCRIPTION = """
@@ -51,14 +51,6 @@ You can also use 🧑🏻‍🚀YI-200K🚀 by cloning this space. 🧬🔬🔍
 Join us : 🌟TeamTonic🌟 is always making cool demos! Join our active builder's🛠️community on 👻Discord: [Discord](https://discord.gg/GWpVpekp) On 🤗Huggingface: [TeamTonic](https://huggingface.co/TeamTonic) & [MultiTransformer](https://huggingface.co/MultiTransformer) On 🌐Github: [Polytonic](https://github.com/tonic-ai) & contribute to 🌟 [PolyGPT](https://github.com/tonic-ai/polygpt-alpha)
 """
 
-DEFAULT_SYSTEM_PROMPT = """
-    You are Yi. You are an AI assistant, you are moderately-polite and give only true information.
-    You carefully provide accurate, factual, thoughtful, nuanced answers, and are brilliant at reasoning. 
-    If you think there might not be a correct answer, you say so. Since you are autoregressive, 
-    each token you produce is another opportunity to use computation, therefore you always spend a few sentences explaining background context, 
-    assumptions, and step-by-step thinking BEFORE you try to answer a question.
-"""
-
 MAX_MAX_NEW_TOKENS = 200000
 DEFAULT_MAX_NEW_TOKENS = 100000
 MAX_INPUT_TOKEN_LENGTH = 100000
@@ -76,12 +68,12 @@ def delete_prev_fn(history=[]):
         message = ''
     return history, message or ''
 
-def generate(message, history_with_input, system_prompt, max_new_tokens, temperature, top_p, top_k):
+def generate(message, history_with_input, max_new_tokens, temperature, top_p, top_k):
     if max_new_tokens > MAX_MAX_NEW_TOKENS:
         raise ValueError
 
     history = history_with_input[:-1]
-    generator = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)
+    generator = run(message, history, max_new_tokens, temperature, top_p, top_k)
     try:
         first_response = next(generator)
         yield history + [(message, first_response)]
@@ -91,12 +83,12 @@ def generate(message, history_with_input, system_prompt, max_new_tokens, tempera
         yield history + [(message, response)]
 
 def process_example(message):
-    generator = generate(message, [], DEFAULT_SYSTEM_PROMPT, 1024, 2.5, 0.95, 900)
+    generator = generate(message, [], 1024, 2.5, 0.95, 900)
     for x in generator:
         pass
     return '', x
 
-def check_input_token_length(message, chat_history, system_prompt):
+def check_input_token_length(message, chat_history):
     input_token_length = len(message) + len(chat_history)
     if input_token_length > MAX_INPUT_TOKEN_LENGTH:
         raise gr.Error(f"The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again.")
@@ -125,7 +117,7 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
     saved_input = gr.State()
 
     with gr.Accordion(label='Advanced options', open=False):
-        system_prompt = gr.Textbox(label='System prompt', value=DEFAULT_SYSTEM_PROMPT, lines=5, interactive=False)
+#       system_prompt = gr.Textbox(label='System prompt', value=DEFAULT_SYSTEM_PROMPT, lines=5, interactive=False)
         max_new_tokens = gr.Slider(label='Max New Tokens', minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)
         temperature = gr.Slider(label='Temperature', minimum=0.1, maximum=4.0, step=0.1, value=0.1)
         top_p = gr.Slider(label='Top-P (nucleus sampling)', minimum=0.05, maximum=1.0, step=0.05, value=0.9)
@@ -145,7 +137,7 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         queue=False,
     ).then(
         fn=check_input_token_length,
-        inputs=[saved_input, chatbot, system_prompt],
+        inputs=[saved_input, chatbot],
         api_name=False,
         queue=False,
     ).success(
@@ -153,7 +145,6 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         inputs=[
             saved_input,
             chatbot,
-            system_prompt,
             max_new_tokens,
             temperature,
             top_p,
@@ -177,7 +168,7 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         queue=False,
     ).then(
         fn=check_input_token_length,
-        inputs=[saved_input, chatbot, system_prompt],
+        inputs=[saved_input, chatbot],
         api_name=False,
         queue=False,
     ).success(
@@ -185,7 +176,6 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         inputs=[
             saved_input,
             chatbot,
-            system_prompt,
             max_new_tokens,
             temperature,
             top_p,
@@ -212,7 +202,6 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         inputs=[
             saved_input,
             chatbot,
-            system_prompt,
             max_new_tokens,
             temperature,
             top_p,