add InCoder and CodeGen tokenizers for comparison (#1)

- add InCoder and CodeGen tokenizers for comparison (6352d45cef72c9eebddc9c78461f965b8c6310b0)


Co-authored-by: loubna ben allal <[email protected]>

app.py

@@ -4,11 +4,26 @@ from transformers import AutoTokenizer
 
 st.set_page_config(page_title="BigCode Tokenizer", page_icon='👩‍💻', layout="wide")
 
-tokenizer = AutoTokenizer.from_pretrained('bigcode/tokenizer', subfolder="digit-custom_punctuation-bytelevel-bpe-py-js-java-50k")
-visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
+models = ["bigcode/tokenizer", "facebook/incoder-6B", "Salesforce/codegen-16B-mono"]
+
+@st.cache()
+def load_tokenizer():
+    if selected_model == "bigcode/tokenizer":
+        tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-custom_punctuation-bytelevel-bpe-py-js-java-50k")
+    else:
+        tokenizer = AutoTokenizer.from_pretrained(selected_model)
+    return tokenizer
+
+col1, col2 = st.columns([1, 2])
+with col1:
+    selected_model = st.selectbox("Select a tokenizer", models, key=1)
 
 text = st.text_area(label="", placeholder="Text to tokenize")
 button_clicked = st.button("Tokenize")
+
+tokenizer = load_tokenizer()
+visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
+
 if text or button_clicked:
     st.write(f"The input was split into {len(tokenizer.tokenize(text))} tokens.")
     st.components.v1.html(visualizer(text), height=1500)