Spaces:

AlGe
/

CompareTokenizers

Running

AlGe commited on Jul 22

Commit

9c925f4

•

1 Parent(s): 0be981f

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -11,9 +11,9 @@ tokenizers = [
     'Xenova/gpt-4o',
     'Xenova/claude-tokenizer',
     'Xenova/mistral-tokenizer-v3',
-    'Xenova/gemma-2-tokenizer',
     'Xenova/llama3-tokenizer-new',
-    'Xenova/Phi-3-mini-4k-instruct-hf'
 ]
 def generate_colored_html(tokens, decoded_tokens):

     'Xenova/gpt-4o',
     'Xenova/claude-tokenizer',
     'Xenova/mistral-tokenizer-v3',
+    #'Xenova/gemma-2-tokenizer',
     'Xenova/llama3-tokenizer-new',
+    #'Xenova/Phi-3-mini-4k-instruct-hf'
 ]
 def generate_colored_html(tokens, decoded_tokens):