Shehryar718 commited on
Commit
61a74e6
1 Parent(s): 359b218

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +84 -84
vocab.json CHANGED
@@ -1,88 +1,88 @@
1
  {
2
- "'": 3,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
- "`": 31,
6
- "|": 80,
7
- "،": 42,
8
- "ؑ": 74,
9
- "ؓ": 10,
10
- "ؔ": 37,
11
- "؛": 11,
12
- "؟": 65,
13
- "ء": 48,
14
- "آ": 75,
15
- "أ": 5,
16
- "ؤ": 35,
17
- "ئ": 15,
18
- "ا": 43,
19
- "ب": 61,
20
- "ت": 32,
21
- "ث": 62,
22
- "ج": 71,
23
- "ح": 53,
24
- "خ": 2,
25
- "د": 8,
26
- "ذ": 47,
27
- "ر": 18,
28
- "ز": 45,
29
- "س": 30,
30
- "ش": 1,
31
- "ص": 40,
32
- "ض": 51,
33
- "ط": 81,
34
- "ظ": 16,
35
- "ع": 68,
36
- "غ": 13,
37
- "ف": 41,
38
- "ق": 66,
39
- "ك": 72,
40
- "ل": 44,
41
- "م": 69,
42
- "ن": 77,
43
- "ه": 12,
44
- "و": 52,
45
- "ى": 22,
46
- "ي": 58,
47
- "ً": 76,
48
- "َ": 20,
49
- "ُ": 33,
50
- "ِ": 82,
51
- "ّ": 50,
52
- "ْ": 70,
53
- "ٓ": 56,
54
- "ٔ": 46,
55
- "ٗ": 59,
56
- "ٰ": 4,
57
- "ٹ": 14,
58
- "پ": 39,
59
- "چ": 29,
60
- "ڈ": 24,
61
- "ڑ": 78,
62
- "ژ": 54,
63
- "ک": 60,
64
- "گ": 19,
65
- "ں": 49,
66
- "ھ": 73,
67
- "ہ": 27,
68
- "ۂ": 0,
69
- "ۃ": 55,
70
- "ی": 57,
71
- "ے": 23,
72
- "ۓ": 7,
73
- "۔": 25,
74
- "’": 21,
75
- "…": 28,
76
- "ﭨ": 83,
77
- "ﮭ": 17,
78
- "ﮯ": 79,
79
- "ﯾ": 63,
80
- "ﷲ": 67,
81
- "ﷺ": 9,
82
- "ﺗ": 26,
83
- "ﺘ": 64,
84
- "ﺩ": 36,
85
- "ﺲ": 34,
86
- "ﻧ": 38,
87
- "ﻮ": 6
88
  }
 
1
  {
2
+ "'": 21,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
+ "`": 19,
6
+ "|": 56,
7
+ "،": 15,
8
+ "ؑ": 25,
9
+ "ؓ": 61,
10
+ "ؔ": 66,
11
+ "؛": 54,
12
+ "؟": 27,
13
+ "ء": 82,
14
+ "آ": 45,
15
+ "أ": 77,
16
+ "ؤ": 68,
17
+ "ئ": 58,
18
+ "ا": 35,
19
+ "ب": 80,
20
+ "ت": 23,
21
+ "ث": 20,
22
+ "ج": 44,
23
+ "ح": 24,
24
+ "خ": 73,
25
+ "د": 60,
26
+ "ذ": 8,
27
+ "ر": 26,
28
+ "ز": 18,
29
+ "س": 3,
30
+ "ش": 13,
31
+ "ص": 48,
32
+ "ض": 75,
33
+ "ط": 31,
34
+ "ظ": 65,
35
+ "ع": 67,
36
+ "غ": 4,
37
+ "ف": 52,
38
+ "ق": 78,
39
+ "ك": 1,
40
+ "ل": 41,
41
+ "م": 28,
42
+ "ن": 29,
43
+ "ه": 64,
44
+ "و": 53,
45
+ "ى": 16,
46
+ "ي": 81,
47
+ "ً": 51,
48
+ "َ": 83,
49
+ "ُ": 57,
50
+ "ِ": 14,
51
+ "ّ": 30,
52
+ "ْ": 79,
53
+ "ٓ": 33,
54
+ "ٔ": 71,
55
+ "ٗ": 43,
56
+ "ٰ": 7,
57
+ "ٹ": 42,
58
+ "پ": 76,
59
+ "چ": 17,
60
+ "ڈ": 38,
61
+ "ڑ": 12,
62
+ "ژ": 72,
63
+ "ک": 74,
64
+ "گ": 62,
65
+ "ں": 10,
66
+ "ھ": 2,
67
+ "ہ": 40,
68
+ "ۂ": 34,
69
+ "ۃ": 5,
70
+ "ی": 9,
71
+ "ے": 0,
72
+ "ۓ": 46,
73
+ "۔": 69,
74
+ "’": 22,
75
+ "…": 47,
76
+ "ﭨ": 50,
77
+ "ﮭ": 70,
78
+ "ﮯ": 63,
79
+ "ﯾ": 59,
80
+ "ﷲ": 11,
81
+ "ﷺ": 55,
82
+ "ﺗ": 37,
83
+ "ﺘ": 6,
84
+ "ﺩ": 32,
85
+ "ﺲ": 39,
86
+ "ﻧ": 36,
87
+ "ﻮ": 49
88
  }