Update app.py

app.py

@@ -9,14 +9,14 @@ model_id = "01-ai/Yi-34B-200K"
 
 os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:50'
 device = "cuda" if torch.cuda.is_available() else "cpu"
-offload_directory = './model_offload'
-if not os.path.exists(offload_directory):
-    os.makedirs(offload_directory)
+# offload_directory = './model_offload'
+# if not os.path.exists(offload_directory):
+#     os.makedirs(offload_directory)
+# model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True, load_in_8bit_fp32_cpu_offload=True, offload_folder=offload_directory, trust_remote_code=True)
+# model = model.to(device)
     
 tokenizer = YiTokenizer(vocab_file="./tokenizer.model")
-model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True, load_in_8bit_fp32_cpu_offload=True, offload_folder=offload_directory, trust_remote_code=True)
-# model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True)
-# model = model.to(device)
+model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True, trust_remote_code=True)
 
 def run(message, chat_history, max_new_tokens=4056, temperature=3.5, top_p=0.9, top_k=800):
     prompt = get_prompt(message, chat_history)