evolve from the myv-rus demo

README.md

@@ -1,12 +1,10 @@
 ---
-title: NLLB Erzya translation demo
-emoji: 🦊
-colorFrom: yellow
-colorTo: red
+title: NLLB-extended translation demo
+emoji: 🐘
+colorFrom: cyan
+colorTo: blue
 sdk: gradio
 sdk_version: 3.46.1
 app_file: app.py
 pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -23,7 +23,7 @@ def translate_wrapper(text, src, trg, by_sentence=True, preprocess=True, random=
 
 
 article = f"""
-This is the demo for a NLLB-200-600M model fine-tuned for translation between Russian and Erzya languages.
+This is the demo for a NLLB-200-600M model fine-tuned for a few (mostly new) languages.
 
 The model itself is available at https://huggingface.co/{MODEL_URL}
 

requirements.txt

@@ -1,4 +1,4 @@
-transformers==4.33
+transformers==4.39
 sentencepiece
 gradio>=3.18.0
 torch

translation.py

@@ -8,13 +8,23 @@ from sacremoses import MosesPunctNormalizer
 from sentence_splitter import SentenceSplitter
 from transformers import AutoModelForSeq2SeqLM, NllbTokenizer
 
-MODEL_URL = "slone/nllb-rus-myv-v1-extvoc"
+MODEL_URL = "slone/nllb-210-v1"
 LANGUAGES = {
-    "Рузонь | Русский | Russian": "rus_Cyrl",
-    "Эрзянь | Эрзянский | Erzya": "myv_Cyrl",
+    "Русский | Russian": "rus_Cyrl",
+    "English | Английский": "eng_Latn",
+    "Azərbaycan | Azerbaijani  | Азербайджанский": "azj_Latn",
+    "Башҡорт | Bashkir | Башкирский": "bak_Cyrl",
+    "Буряад | Buryat | Бурятский": "bxr_Cyrl",
+    "Чӑваш | Chuvash | Чувашский": "chv_Cyrl",
+    "Хакас | Khakas | Хакасский": "kjh_Cyrl",
+    "Къарачай-малкъар | Karachay-Balkar | Карачаево-балкарский": "krc_Cyrl",
+    "Марий | Meadow Mari | Марийский": "mhr_Cyrl",
+    "Эрзянь | Erzya | Эрзянский": "myv_Cyrl",
+    "Татар | Tatar | Татарский": "tat_Cyrl",
+    "Тыва | Тувинский | Tuvan ": "tyv_Cyrl",
 }
 L1 = "rus_Cyrl"
-L2 = "myv_Cyrl"
+L2 = "eng_Latn"
 
 
 def get_non_printing_char_replacer(replace_by: str = " ") -> tp.Callable[[str], str]:
@@ -54,31 +64,6 @@ class TextPreprocessor:
         return clean
 
 
-def fix_tokenizer(tokenizer, new_lang=L2):
-    """Add a new language token to the tokenizer vocabulary
-    (this should be done each time after its initialization)
-    """
-    old_len = len(tokenizer) - int(new_lang in tokenizer.added_tokens_encoder)
-    tokenizer.lang_code_to_id[new_lang] = old_len - 1
-    tokenizer.id_to_lang_code[old_len - 1] = new_lang
-    # always move "mask" to the last position
-    tokenizer.fairseq_tokens_to_ids["<mask>"] = (
-        len(tokenizer.sp_model)
-        + len(tokenizer.lang_code_to_id)
-        + tokenizer.fairseq_offset
-    )
-
-    tokenizer.fairseq_tokens_to_ids.update(tokenizer.lang_code_to_id)
-    tokenizer.fairseq_ids_to_tokens = {
-        v: k for k, v in tokenizer.fairseq_tokens_to_ids.items()
-    }
-    if new_lang not in tokenizer._additional_special_tokens:
-        tokenizer._additional_special_tokens.append(new_lang)
-    # clear the added token encoder; otherwise a new token may end up there by mistake
-    tokenizer.added_tokens_encoder = {}
-    tokenizer.added_tokens_decoder = {}
-
-
 def sentenize_with_fillers(text, splitter, fix_double_space=True, ignore_errors=False):
     """Apply a sentence splitter and return the sentences and all separators before and after them"""
     if fix_double_space:
@@ -104,7 +89,6 @@ class Translator:
         if torch.cuda.is_available():
             self.model.cuda()
         self.tokenizer = NllbTokenizer.from_pretrained(MODEL_URL)
-        fix_tokenizer(self.tokenizer)
 
         self.splitter = SentenceSplitter("ru")
         self.preprocessor = TextPreprocessor()