Update app.py

app.py

@@ -78,7 +78,7 @@ class S3ModelLoader:
                 if config.pad_token_id is not None:
                     tokenizer.pad_token_id = config.pad_token_id
                 else:
-                    tokenizer.pad_token_id = 0
+                    tokenizer.pad_token_id = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else 0
 
             logging.info(f"Loaded {model_name} from S3 successfully.")
             return model, tokenizer
@@ -93,7 +93,7 @@ class S3ModelLoader:
                     if config.pad_token_id is not None:
                         tokenizer.pad_token_id = config.pad_token_id
                     else:
-                        tokenizer.pad_token_id = 0
+                        tokenizer.pad_token_id = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else 0
 
                 logging.info(f"Downloaded {model_name} successfully.")
                 logging.info(f"Saving {model_name} to S3...")
@@ -128,7 +128,7 @@ async def generate(request: Request, body: GenerateRequest):
                 do_sample=validated_body.do_sample,
                 num_return_sequences=validated_body.num_return_sequences,
                 stopping_criteria=StoppingCriteriaList(
-                    [lambda _, outputs: tokenizer.decode(outputs[0][-1]) in validated_body.stop_sequences] if validated_body.stop_sequences else []
+                    [lambda _, outputs: tokenizer.decode(outputs[0][-1], skip_special_tokens=True) in validated_body.stop_sequences] if validated_body.stop_sequences else []
                 )
             )