Upload joycaption.py

joycaption.py

@@ -194,8 +194,7 @@ def load_text_model(model_name: str=MODEL_PATH, gguf_file: Union[str, None]=None
             return
 
         print("Loading tokenizer")
-        if gguf_file: tokenizer = AutoTokenizer.from_pretrained(model_name, gguf_file=gguf_file, use_fast=True, legacy=False)
-        else: tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, legacy=False)
+        tokenizer = AutoTokenizer.from_pretrained(CHECKPOINT_PATH / "text_model", use_fast=True)
         assert isinstance(tokenizer, PreTrainedTokenizer) or isinstance(tokenizer, PreTrainedTokenizerFast), f"Tokenizer is of type {type(tokenizer)}"
  
         print(f"Loading LLM: {model_name}")