asd

app.py

@@ -368,6 +368,13 @@ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForSeque
 from sentence_splitter import SentenceSplitter
 from itertools import product
 
+# Ensure sentencepiece is installed
+try:
+    import sentencepiece
+except ImportError:
+    import subprocess
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "sentencepiece"])
+
 # Get the Hugging Face token from environment variable
 hf_token = os.getenv('HF_TOKEN')
 
@@ -377,7 +384,7 @@ print(f"Using device: {device}")
 
 # Initialize paraphraser model and tokenizer
 paraphraser_model_name = "ramsrigouthamg/t5-large-paraphraser-diverse-high-quality"
-paraphraser_tokenizer = AutoTokenizer.from_pretrained(paraphraser_model_name)
+paraphraser_tokenizer = AutoTokenizer.from_pretrained(paraphraser_model_name, use_fast=False)
 paraphraser_model = AutoModelForSeq2SeqLM.from_pretrained(paraphraser_model_name).to(device)
 
 # Initialize classifier model and tokenizer