Shehryar718 commited on
Commit
b77f97f
1 Parent(s): f91cb75

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +83 -83
vocab.json CHANGED
@@ -1,88 +1,88 @@
1
  {
2
- "'": 50,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
- "`": 13,
6
- "|": 66,
7
- "،": 76,
8
- "ؑ": 74,
9
- "ؓ": 40,
10
- "ؔ": 64,
11
- "؛": 75,
12
- "؟": 14,
13
- "ء": 81,
14
- "آ": 57,
15
- "أ": 39,
16
- "ؤ": 78,
17
- "ئ": 26,
18
- "ا": 51,
19
- "ب": 77,
20
- "ت": 80,
21
- "ث": 69,
22
- "ج": 59,
23
- "ح": 58,
24
- "خ": 24,
25
- "د": 27,
26
- "ذ": 49,
27
- "ر": 20,
28
- "ز": 41,
29
- "س": 62,
30
- "ش": 17,
31
- "ص": 55,
32
- "ض": 54,
33
- "ط": 56,
34
- "ظ": 42,
35
- "ع": 9,
36
- "غ": 32,
37
- "ف": 2,
38
- "ق": 43,
39
- "ك": 47,
40
- "ل": 36,
41
- "م": 63,
42
- "ن": 72,
43
- "ه": 44,
44
- "و": 6,
45
- "ى": 68,
46
- "ي": 53,
47
- "ً": 29,
48
- "َ": 67,
49
- "ُ": 8,
50
- "ِ": 83,
51
- "ّ": 30,
52
- "ْ": 23,
53
- "ٓ": 0,
54
- "ٔ": 33,
55
- "ٗ": 15,
56
- "ٰ": 4,
57
- "ٹ": 21,
58
- "پ": 19,
59
- "چ": 35,
60
- "ڈ": 61,
61
- "ڑ": 38,
62
  "ژ": 16,
63
- "ک": 52,
64
- "گ": 70,
65
- "ں": 1,
66
- "ھ": 22,
67
- "ہ": 28,
68
- "ۂ": 79,
69
- "ۃ": 5,
70
- "ی": 10,
71
- "ے": 7,
72
- "ۓ": 25,
73
- "۔": 11,
74
- "’": 71,
75
- "…": 31,
76
- "ﭨ": 37,
77
- "ﮭ": 73,
78
- "ﮯ": 46,
79
- "ﯾ": 48,
80
- "ﷲ": 12,
81
- "ﷺ": 45,
82
- "ﺗ": 3,
83
- "ﺘ": 18,
84
- "ﺩ": 65,
85
- "ﺲ": 82,
86
- "ﻧ": 60,
87
- "ﻮ": 34
88
  }
 
1
  {
2
+ "'": 65,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
+ "`": 34,
6
+ "|": 56,
7
+ "،": 13,
8
+ "ؑ": 81,
9
+ "ؓ": 54,
10
+ "ؔ": 7,
11
+ "؛": 60,
12
+ "؟": 23,
13
+ "ء": 30,
14
+ "آ": 8,
15
+ "أ": 2,
16
+ "ؤ": 33,
17
+ "ئ": 4,
18
+ "ا": 22,
19
+ "ب": 14,
20
+ "ت": 25,
21
+ "ث": 28,
22
+ "ج": 57,
23
+ "ح": 66,
24
+ "خ": 64,
25
+ "د": 5,
26
+ "ذ": 29,
27
+ "ر": 41,
28
+ "ز": 80,
29
+ "س": 20,
30
+ "ش": 68,
31
+ "ص": 62,
32
+ "ض": 24,
33
+ "ط": 21,
34
+ "ظ": 11,
35
+ "ع": 70,
36
+ "غ": 19,
37
+ "ف": 53,
38
+ "ق": 38,
39
+ "ك": 52,
40
+ "ل": 69,
41
+ "م": 3,
42
+ "ن": 82,
43
+ "ه": 61,
44
+ "و": 73,
45
+ "ى": 51,
46
+ "ي": 26,
47
+ "ً": 78,
48
+ "َ": 77,
49
+ "ُ": 43,
50
+ "ِ": 49,
51
+ "ّ": 36,
52
+ "ْ": 18,
53
+ "ٓ": 76,
54
+ "ٔ": 12,
55
+ "ٗ": 58,
56
+ "ٰ": 59,
57
+ "ٹ": 45,
58
+ "پ": 27,
59
+ "چ": 31,
60
+ "ڈ": 55,
61
+ "ڑ": 42,
62
  "ژ": 16,
63
+ "ک": 35,
64
+ "گ": 40,
65
+ "ں": 75,
66
+ "ھ": 39,
67
+ "ہ": 6,
68
+ "ۂ": 74,
69
+ "ۃ": 1,
70
+ "ی": 48,
71
+ "ے": 9,
72
+ "ۓ": 37,
73
+ "۔": 71,
74
+ "’": 10,
75
+ "…": 83,
76
+ "ﭨ": 44,
77
+ "ﮭ": 72,
78
+ "ﮯ": 79,
79
+ "ﯾ": 15,
80
+ "ﷲ": 67,
81
+ "ﷺ": 17,
82
+ "ﺗ": 63,
83
+ "ﺘ": 46,
84
+ "ﺩ": 50,
85
+ "ﺲ": 47,
86
+ "ﻧ": 32,
87
+ "ﻮ": 0
88
  }