Upload tokenizer

tokenization_phylogpn.py

@@ -4,7 +4,7 @@ from transformers import PreTrainedTokenizer
 class PhyloGPNTokenizer(PreTrainedTokenizer):
     model_input_names = ["input_ids"]
 
-    def __init__(self, model_max_length: int = None, unk_token="N", pad_token="-", bos_token=None, eos_token=None, sep_token=None, cls_token=None, mask_token=None, **kwargs):
+    def __init__(self, model_max_length: int = None, unk_token="N", pad_token="-", bos_token=None, eos_token=None, sep_token=None, cls_token=None, mask_token=None, split_special_tokens=True, **kwargs):
         self.model_max_length = model_max_length
         self._vocab = {k: v for v, k in enumerate("ACGTN-")}
 
@@ -19,6 +19,7 @@ class PhyloGPNTokenizer(PreTrainedTokenizer):
             sep_token=sep_token,
             cls_token=cls_token,
             mask_token=mask_token,
+            split_special_tokens=split_special_tokens,
             add_prefix_space=add_prefix_space,
             padding_side=padding_side,
             **kwargs,

tokenizer_config.json

@@ -34,6 +34,7 @@
   "pad_token": "-",
   "padding_side": "right",
   "sep_token": null,
+  "split_special_tokens": true,
   "tokenizer_class": "PhyloGPNTokenizer",
   "unk_token": "N"
 }