Update README.md

README.md

@@ -2,6 +2,18 @@
 
 This is a roBERTa-base model trained on ~58M tweets, described and evaluated in the [_TweetEval_ benchmark (Findings of EMNLP 2020)](https://arxiv.org/pdf/2010.12421.pdf). To evaluate this and other LMs on Twitter-specific data, please refer to the [Tweeteval official repository](https://github.com/cardiffnlp/tweeteval).
 
+## Preprocess Text 
+Replace usernames and links for placeholders: "@user" and "http".
+```python
+def preprocess(text):
+    new_text = []
+    for t in text.split(" "):
+        t = '@user' if t.startswith('@') and len(t) > 1 else t
+        t = 'http' if t.startswith('http') else t
+        new_text.append(t)
+    return " ".join(new_text)
+```
+
 ## Example Masked Language Model 
 
 ```python
@@ -23,8 +35,9 @@ texts = [
  "I am so <mask> 😢" 
 ]
 for text in texts:
-    print(f"{'-'*30}\n{text}")
-    candidates = fill_mask(text)
+    t = preprocess(text)
+    print(f"{'-'*30}\n{t}")
+    candidates = fill_mask(t)
     print_candidates()
 ```
 
@@ -55,7 +68,7 @@ import numpy as np
 
 MODEL = "cardiffnlp/twitter-roberta-base"
 text = "Good night 😊"
-
+text = preprocess(text)
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 
 # Pytorch