n

run_speech_recognition_ctc.py

@@ -511,7 +511,6 @@ def main():
         tokenizer_kwargs = {
             "config": config if config.tokenizer_class is not None else None,
             "tokenizer_type": config.model_type if config.tokenizer_class is None else None,
-            "bos_token": "<s>",
             "unk_token": unk_token,
             "pad_token": pad_token,
             "word_delimiter_token": word_delimiter_token,
@@ -522,11 +521,10 @@ def main():
     # one local process can concurrently download model & vocab.
 
     # load feature_extractor and tokenizer
-    tokenizer = AutoTokenizer.from_pretrained(
-        tokenizer_name_or_path,
-        use_auth_token=data_args.use_auth_token,
-        **tokenizer_kwargs,
-    )
+    tokenizer = Wav2Vec2CTCTokenizer(tokenizer_name_or_path, 
+                                     use_auth_token=data_args.use_auth_token,
+                                     **tokenizer_kwargs,
+                                    )
     feature_extractor = AutoFeatureExtractor.from_pretrained(
         model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token
     )