Upload tokens.txt
Browse files- data/lang_bpe_500/tokens.txt +500 -0
data/lang_bpe_500/tokens.txt
ADDED
@@ -0,0 +1,500 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<blk> 0
|
2 |
+
<sos/eos> 1
|
3 |
+
<unk> 2
|
4 |
+
S 3
|
5 |
+
▁THE 4
|
6 |
+
▁A 5
|
7 |
+
T 6
|
8 |
+
▁AND 7
|
9 |
+
ED 8
|
10 |
+
▁OF 9
|
11 |
+
▁TO 10
|
12 |
+
E 11
|
13 |
+
D 12
|
14 |
+
N 13
|
15 |
+
ING 14
|
16 |
+
▁IN 15
|
17 |
+
Y 16
|
18 |
+
M 17
|
19 |
+
C 18
|
20 |
+
▁I 19
|
21 |
+
A 20
|
22 |
+
P 21
|
23 |
+
▁HE 22
|
24 |
+
R 23
|
25 |
+
O 24
|
26 |
+
L 25
|
27 |
+
RE 26
|
28 |
+
I 27
|
29 |
+
U 28
|
30 |
+
ER 29
|
31 |
+
▁IT 30
|
32 |
+
LY 31
|
33 |
+
▁THAT 32
|
34 |
+
▁WAS 33
|
35 |
+
▁ 34
|
36 |
+
▁S 35
|
37 |
+
AR 36
|
38 |
+
▁BE 37
|
39 |
+
F 38
|
40 |
+
▁C 39
|
41 |
+
IN 40
|
42 |
+
B 41
|
43 |
+
▁FOR 42
|
44 |
+
OR 43
|
45 |
+
LE 44
|
46 |
+
' 45
|
47 |
+
▁HIS 46
|
48 |
+
▁YOU 47
|
49 |
+
AL 48
|
50 |
+
▁RE 49
|
51 |
+
V 50
|
52 |
+
▁B 51
|
53 |
+
G 52
|
54 |
+
RI 53
|
55 |
+
▁E 54
|
56 |
+
▁WITH 55
|
57 |
+
▁T 56
|
58 |
+
▁AS 57
|
59 |
+
LL 58
|
60 |
+
▁P 59
|
61 |
+
▁HER 60
|
62 |
+
ST 61
|
63 |
+
▁HAD 62
|
64 |
+
▁SO 63
|
65 |
+
▁F 64
|
66 |
+
W 65
|
67 |
+
CE 66
|
68 |
+
▁IS 67
|
69 |
+
ND 68
|
70 |
+
▁NOT 69
|
71 |
+
TH 70
|
72 |
+
▁BUT 71
|
73 |
+
EN 72
|
74 |
+
▁SHE 73
|
75 |
+
▁ON 74
|
76 |
+
VE 75
|
77 |
+
ON 76
|
78 |
+
SE 77
|
79 |
+
▁DE 78
|
80 |
+
UR 79
|
81 |
+
▁G 80
|
82 |
+
CH 81
|
83 |
+
K 82
|
84 |
+
TER 83
|
85 |
+
▁AT 84
|
86 |
+
IT 85
|
87 |
+
▁ME 86
|
88 |
+
RO 87
|
89 |
+
NE 88
|
90 |
+
RA 89
|
91 |
+
ES 90
|
92 |
+
IL 91
|
93 |
+
NG 92
|
94 |
+
IC 93
|
95 |
+
▁NO 94
|
96 |
+
▁HIM 95
|
97 |
+
ENT 96
|
98 |
+
IR 97
|
99 |
+
▁WE 98
|
100 |
+
H 99
|
101 |
+
▁DO 100
|
102 |
+
▁ALL 101
|
103 |
+
▁HAVE 102
|
104 |
+
LO 103
|
105 |
+
▁BY 104
|
106 |
+
▁MY 105
|
107 |
+
▁MO 106
|
108 |
+
▁THIS 107
|
109 |
+
LA 108
|
110 |
+
▁ST 109
|
111 |
+
▁WHICH 110
|
112 |
+
▁CON 111
|
113 |
+
▁THEY 112
|
114 |
+
CK 113
|
115 |
+
TE 114
|
116 |
+
▁SAID 115
|
117 |
+
▁FROM 116
|
118 |
+
▁GO 117
|
119 |
+
▁WHO 118
|
120 |
+
▁TH 119
|
121 |
+
▁OR 120
|
122 |
+
▁D 121
|
123 |
+
▁W 122
|
124 |
+
VER 123
|
125 |
+
LI 124
|
126 |
+
▁SE 125
|
127 |
+
▁ONE 126
|
128 |
+
▁CA 127
|
129 |
+
▁AN 128
|
130 |
+
▁LA 129
|
131 |
+
▁WERE 130
|
132 |
+
EL 131
|
133 |
+
▁HA 132
|
134 |
+
▁MAN 133
|
135 |
+
▁FA 134
|
136 |
+
▁EX 135
|
137 |
+
AD 136
|
138 |
+
▁SU 137
|
139 |
+
RY 138
|
140 |
+
▁MI 139
|
141 |
+
AT 140
|
142 |
+
▁BO 141
|
143 |
+
▁WHEN 142
|
144 |
+
AN 143
|
145 |
+
THER 144
|
146 |
+
PP 145
|
147 |
+
ATION 146
|
148 |
+
▁FI 147
|
149 |
+
▁WOULD 148
|
150 |
+
▁PRO 149
|
151 |
+
OW 150
|
152 |
+
ET 151
|
153 |
+
▁O 152
|
154 |
+
▁THERE 153
|
155 |
+
▁HO 154
|
156 |
+
ION 155
|
157 |
+
▁WHAT 156
|
158 |
+
▁FE 157
|
159 |
+
▁PA 158
|
160 |
+
US 159
|
161 |
+
MENT 160
|
162 |
+
▁MA 161
|
163 |
+
UT 162
|
164 |
+
▁OUT 163
|
165 |
+
▁THEIR 164
|
166 |
+
▁IF 165
|
167 |
+
▁LI 166
|
168 |
+
▁K 167
|
169 |
+
▁WILL 168
|
170 |
+
▁ARE 169
|
171 |
+
ID 170
|
172 |
+
▁RO 171
|
173 |
+
DE 172
|
174 |
+
TION 173
|
175 |
+
▁WA 174
|
176 |
+
PE 175
|
177 |
+
▁UP 176
|
178 |
+
▁SP 177
|
179 |
+
▁PO 178
|
180 |
+
IGHT 179
|
181 |
+
▁UN 180
|
182 |
+
RU 181
|
183 |
+
▁LO 182
|
184 |
+
AS 183
|
185 |
+
OL 184
|
186 |
+
▁LE 185
|
187 |
+
▁BEEN 186
|
188 |
+
▁SH 187
|
189 |
+
▁RA 188
|
190 |
+
▁SEE 189
|
191 |
+
KE 190
|
192 |
+
UL 191
|
193 |
+
TED 192
|
194 |
+
▁SA 193
|
195 |
+
UN 194
|
196 |
+
UND 195
|
197 |
+
ANT 196
|
198 |
+
▁NE 197
|
199 |
+
IS 198
|
200 |
+
▁THEM 199
|
201 |
+
CI 200
|
202 |
+
GE 201
|
203 |
+
▁COULD 202
|
204 |
+
▁DIS 203
|
205 |
+
OM 204
|
206 |
+
ISH 205
|
207 |
+
HE 206
|
208 |
+
EST 207
|
209 |
+
▁SOME 208
|
210 |
+
ENCE 209
|
211 |
+
ITY 210
|
212 |
+
IVE 211
|
213 |
+
▁US 212
|
214 |
+
▁MORE 213
|
215 |
+
▁EN 214
|
216 |
+
ARD 215
|
217 |
+
ATE 216
|
218 |
+
▁YOUR 217
|
219 |
+
▁INTO 218
|
220 |
+
▁KNOW 219
|
221 |
+
▁CO 220
|
222 |
+
ANCE 221
|
223 |
+
▁TIME 222
|
224 |
+
▁WI 223
|
225 |
+
▁YE 224
|
226 |
+
AGE 225
|
227 |
+
▁NOW 226
|
228 |
+
TI 227
|
229 |
+
FF 228
|
230 |
+
ABLE 229
|
231 |
+
▁VERY 230
|
232 |
+
▁LIKE 231
|
233 |
+
AM 232
|
234 |
+
HI 233
|
235 |
+
Z 234
|
236 |
+
▁OTHER 235
|
237 |
+
▁THAN 236
|
238 |
+
▁LITTLE 237
|
239 |
+
▁DID 238
|
240 |
+
▁LOOK 239
|
241 |
+
TY 240
|
242 |
+
ERS 241
|
243 |
+
▁CAN 242
|
244 |
+
▁CHA 243
|
245 |
+
▁AR 244
|
246 |
+
X 245
|
247 |
+
FUL 246
|
248 |
+
UGH 247
|
249 |
+
▁BA 248
|
250 |
+
▁DAY 249
|
251 |
+
▁ABOUT 250
|
252 |
+
TEN 251
|
253 |
+
IM 252
|
254 |
+
▁ANY 253
|
255 |
+
▁PRE 254
|
256 |
+
▁OVER 255
|
257 |
+
IES 256
|
258 |
+
NESS 257
|
259 |
+
ME 258
|
260 |
+
BLE 259
|
261 |
+
▁M 260
|
262 |
+
ROW 261
|
263 |
+
▁HAS 262
|
264 |
+
▁GREAT 263
|
265 |
+
▁VI 264
|
266 |
+
TA 265
|
267 |
+
▁AFTER 266
|
268 |
+
PER 267
|
269 |
+
▁AGAIN 268
|
270 |
+
HO 269
|
271 |
+
SH 270
|
272 |
+
▁UPON 271
|
273 |
+
▁DI 272
|
274 |
+
▁HAND 273
|
275 |
+
▁COM 274
|
276 |
+
IST 275
|
277 |
+
TURE 276
|
278 |
+
▁STA 277
|
279 |
+
▁THEN 278
|
280 |
+
▁SHOULD 279
|
281 |
+
▁GA 280
|
282 |
+
OUS 281
|
283 |
+
OUR 282
|
284 |
+
▁WELL 283
|
285 |
+
▁ONLY 284
|
286 |
+
MAN 285
|
287 |
+
▁GOOD 286
|
288 |
+
▁TWO 287
|
289 |
+
▁MAR 288
|
290 |
+
▁SAY 289
|
291 |
+
▁HU 290
|
292 |
+
TING 291
|
293 |
+
▁OUR 292
|
294 |
+
RESS 293
|
295 |
+
▁DOWN 294
|
296 |
+
IOUS 295
|
297 |
+
▁BEFORE 296
|
298 |
+
▁DA 297
|
299 |
+
▁NA 298
|
300 |
+
QUI 299
|
301 |
+
▁MADE 300
|
302 |
+
▁EVERY 301
|
303 |
+
▁OLD 302
|
304 |
+
▁EVEN 303
|
305 |
+
IG 304
|
306 |
+
▁COME 305
|
307 |
+
▁GRA 306
|
308 |
+
▁RI 307
|
309 |
+
▁LONG 308
|
310 |
+
OT 309
|
311 |
+
SIDE 310
|
312 |
+
WARD 311
|
313 |
+
▁FO 312
|
314 |
+
▁WHERE 313
|
315 |
+
MO 314
|
316 |
+
LESS 315
|
317 |
+
▁SC 316
|
318 |
+
▁MUST 317
|
319 |
+
▁NEVER 318
|
320 |
+
▁HOW 319
|
321 |
+
▁CAME 320
|
322 |
+
▁SUCH 321
|
323 |
+
▁RU 322
|
324 |
+
▁TAKE 323
|
325 |
+
▁WO 324
|
326 |
+
▁CAR 325
|
327 |
+
UM 326
|
328 |
+
AK 327
|
329 |
+
▁THINK 328
|
330 |
+
▁MUCH 329
|
331 |
+
▁MISTER 330
|
332 |
+
▁MAY 331
|
333 |
+
▁JO 332
|
334 |
+
▁WAY 333
|
335 |
+
▁COMP 334
|
336 |
+
▁THOUGHT 335
|
337 |
+
▁STO 336
|
338 |
+
▁MEN 337
|
339 |
+
▁BACK 338
|
340 |
+
▁DON 339
|
341 |
+
J 340
|
342 |
+
▁LET 341
|
343 |
+
▁TRA 342
|
344 |
+
▁FIRST 343
|
345 |
+
▁JUST 344
|
346 |
+
▁VA 345
|
347 |
+
▁OWN 346
|
348 |
+
▁PLA 347
|
349 |
+
▁MAKE 348
|
350 |
+
ATED 349
|
351 |
+
▁HIMSELF 350
|
352 |
+
▁WENT 351
|
353 |
+
▁PI 352
|
354 |
+
GG 353
|
355 |
+
RING 354
|
356 |
+
▁DU 355
|
357 |
+
▁MIGHT 356
|
358 |
+
▁PART 357
|
359 |
+
▁GIVE 358
|
360 |
+
▁IMP 359
|
361 |
+
▁BU 360
|
362 |
+
▁PER 361
|
363 |
+
▁PLACE 362
|
364 |
+
▁HOUSE 363
|
365 |
+
▁THROUGH 364
|
366 |
+
IAN 365
|
367 |
+
▁SW 366
|
368 |
+
▁UNDER 367
|
369 |
+
QUE 368
|
370 |
+
▁AWAY 369
|
371 |
+
▁LOVE 370
|
372 |
+
QUA 371
|
373 |
+
▁LIFE 372
|
374 |
+
▁GET 373
|
375 |
+
▁WITHOUT 374
|
376 |
+
▁PASS 375
|
377 |
+
▁TURN 376
|
378 |
+
IGN 377
|
379 |
+
▁HEAD 378
|
380 |
+
▁MOST 379
|
381 |
+
▁THOSE 380
|
382 |
+
▁SHALL 381
|
383 |
+
▁EYES 382
|
384 |
+
▁COL 383
|
385 |
+
▁STILL 384
|
386 |
+
▁NIGHT 385
|
387 |
+
▁NOTHING 386
|
388 |
+
ITION 387
|
389 |
+
HA 388
|
390 |
+
▁TELL 389
|
391 |
+
▁WORK 390
|
392 |
+
▁LAST 391
|
393 |
+
▁NEW 392
|
394 |
+
▁FACE 393
|
395 |
+
▁HI 394
|
396 |
+
▁WORD 395
|
397 |
+
▁FOUND 396
|
398 |
+
▁COUNT 397
|
399 |
+
▁OB 398
|
400 |
+
▁WHILE 399
|
401 |
+
▁SHA 400
|
402 |
+
▁MEAN 401
|
403 |
+
▁SAW 402
|
404 |
+
▁PEOPLE 403
|
405 |
+
▁FRIEND 404
|
406 |
+
▁THREE 405
|
407 |
+
▁ROOM 406
|
408 |
+
▁SAME 407
|
409 |
+
▁THOUGH 408
|
410 |
+
▁RIGHT 409
|
411 |
+
▁CHILD 410
|
412 |
+
▁FATHER 411
|
413 |
+
▁ANOTHER 412
|
414 |
+
▁HEART 413
|
415 |
+
▁WANT 414
|
416 |
+
▁TOOK 415
|
417 |
+
OOK 416
|
418 |
+
▁LIGHT 417
|
419 |
+
▁MISSUS 418
|
420 |
+
▁OPEN 419
|
421 |
+
▁JU 420
|
422 |
+
▁ASKED 421
|
423 |
+
PORT 422
|
424 |
+
▁LEFT 423
|
425 |
+
▁JA 424
|
426 |
+
▁WORLD 425
|
427 |
+
▁HOME 426
|
428 |
+
▁WHY 427
|
429 |
+
▁ALWAYS 428
|
430 |
+
▁ANSWER 429
|
431 |
+
▁SEEMED 430
|
432 |
+
▁SOMETHING 431
|
433 |
+
▁GIRL 432
|
434 |
+
▁BECAUSE 433
|
435 |
+
▁NAME 434
|
436 |
+
▁TOLD 435
|
437 |
+
▁NI 436
|
438 |
+
▁HIGH 437
|
439 |
+
IZE 438
|
440 |
+
▁WOMAN 439
|
441 |
+
▁FOLLOW 440
|
442 |
+
▁RETURN 441
|
443 |
+
▁KNEW 442
|
444 |
+
▁EACH 443
|
445 |
+
▁KIND 444
|
446 |
+
▁JE 445
|
447 |
+
▁ACT 446
|
448 |
+
▁LU 447
|
449 |
+
▁CERTAIN 448
|
450 |
+
▁YEARS 449
|
451 |
+
▁QUITE 450
|
452 |
+
▁APPEAR 451
|
453 |
+
▁BETTER 452
|
454 |
+
▁HALF 453
|
455 |
+
▁PRESENT 454
|
456 |
+
▁PRINCE 455
|
457 |
+
SHIP 456
|
458 |
+
▁ALSO 457
|
459 |
+
▁BEGAN 458
|
460 |
+
▁HAVING 459
|
461 |
+
▁ENOUGH 460
|
462 |
+
▁PERSON 461
|
463 |
+
▁LADY 462
|
464 |
+
▁WHITE 463
|
465 |
+
▁COURSE 464
|
466 |
+
▁VOICE 465
|
467 |
+
▁SPEAK 466
|
468 |
+
▁POWER 467
|
469 |
+
▁MORNING 468
|
470 |
+
▁BETWEEN 469
|
471 |
+
▁AMONG 470
|
472 |
+
▁KEEP 471
|
473 |
+
▁WALK 472
|
474 |
+
▁MATTER 473
|
475 |
+
▁TEA 474
|
476 |
+
▁BELIEVE 475
|
477 |
+
▁SMALL 476
|
478 |
+
▁TALK 477
|
479 |
+
▁FELT 478
|
480 |
+
▁HORSE 479
|
481 |
+
▁MYSELF 480
|
482 |
+
▁SIX 481
|
483 |
+
▁HOWEVER 482
|
484 |
+
▁FULL 483
|
485 |
+
▁HERSELF 484
|
486 |
+
▁POINT 485
|
487 |
+
▁STOOD 486
|
488 |
+
▁HUNDRED 487
|
489 |
+
▁ALMOST 488
|
490 |
+
▁SINCE 489
|
491 |
+
▁LARGE 490
|
492 |
+
▁LEAVE 491
|
493 |
+
▁PERHAPS 492
|
494 |
+
▁DARK 493
|
495 |
+
▁SUDDEN 494
|
496 |
+
▁REPLIED 495
|
497 |
+
▁ANYTHING 496
|
498 |
+
▁WONDER 497
|
499 |
+
▁UNTIL 498
|
500 |
+
Q 499
|