phi3

app.py

@@ -51,7 +51,7 @@ model_name = get_model_name_from_path(args.model_path)
 model_kwargs = {
     "trust_remote_code": True,
     "torch_dtype": torch.bfloat16,
-    "attn_implementation": "flash_attention_2"
+    "attn_implementation": "eager"
 }
 tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, device_map="cuda:0", **model_kwargs)
 our_chatbot = None
@@ -155,11 +155,13 @@ def generate(state, imagebox, textbox, image_process_mode, temperature, top_p, m
         streamer=streamer,
         use_cache=True,
         pad_token_id=tokenizer.eos_token_id,
+        eos_token_id=[32007],
         **image_args
     ))
     thread.start()
     generated_text = ''
     for new_text in streamer:
+        new_text = new_text.replace('<|end|>', "")
         generated_text += new_text
         if generated_text.endswith(stop_str):
             generated_text = generated_text[:-len(stop_str)]