cym2784841400
/

Qwen2.5-Math-7B-Instruct-Custom-Handler

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

cyqm commited on Nov 22, 2024

Commit

15cdada

·

1 Parent(s): 247c739

Update handler

Files changed (1) hide show

handler.py +6 -1

handler.py CHANGED Viewed

@@ -46,7 +46,12 @@ class EndpointHandler:
         model_inputs = self.tokenizer([text], return_tensors="pt").to("cuda")
         time_start = time.time()
-        generated_ids = self.model.generate(**model_inputs, max_new_tokens=max_new_tokens)
         time_end = time.time()
         num_new_tokens = len(generated_ids[0]) - len(model_inputs[0])

         model_inputs = self.tokenizer([text], return_tensors="pt").to("cuda")
         time_start = time.time()
+        generated_ids = self.model.generate(
+            **model_inputs,
+            max_new_tokens=max_new_tokens,
+            temperature=1.0,
+            **parameters
+        )
         time_end = time.time()
         num_new_tokens = len(generated_ids[0]) - len(model_inputs[0])