Update handler.py

handler.py

@@ -45,7 +45,7 @@ class EndpointHandler:
         inputs = self.tokenizer(prompt+' \n  >> <assistant>:', return_tensors="pt").to(device)
         inputs = {k: v.to('cuda') for k, v in inputs.items()}
 
-        output = self.inference_model.generate(input_ids=inputs["input_ids"],pad_token_id=self.tokenizer.pad_token_id, max_new_tokens=256, do_sample=True, temperature=0.9, top_p=0.9, repetition_penalty=1.5, early_stopping=True, length_penalty = -0.3, num_beams=5, num_return_sequences=1)
+        output = self.inference_model.generate(input_ids=inputs["input_ids"],pad_token_id=self.tokenizer.pad_token_id, max_new_tokens=64, do_sample=True, temperature=0.9, top_p=0.9, repetition_penalty=1.5, early_stopping=True, length_penalty = -0.3, num_beams=5, num_return_sequences=1)
         response_raw = self.tokenizer.batch_decode(output.detach().cpu().numpy(), skip_special_tokens=True)
         response_ls = response_raw[0].split('>>')
         response_ = response_ls[1].split('<assistant>:')[1]