VictorSanh commited on
Commit
6e9baf5
1 Parent(s): 1cb5729

tiny fix with connector update

Browse files
special_tokens_map.json CHANGED
@@ -13,13 +13,6 @@
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
- },
17
- {
18
- "content": "<end_of_utterance>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
  }
24
  ],
25
  "bos_token": {
 
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
 
 
 
 
 
 
 
16
  }
17
  ],
18
  "bos_token": {
tokenizer.json CHANGED
@@ -47,15 +47,6 @@
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
- },
51
- {
52
- "id": 32002,
53
- "content": "<end_of_utterance>",
54
- "single_word": false,
55
- "lstrip": false,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": true
59
  }
60
  ],
61
  "normalizer": {
 
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
 
 
 
 
 
 
 
 
 
50
  }
51
  ],
52
  "normalizer": {
tokenizer_config.json CHANGED
@@ -41,20 +41,11 @@
41
  "rstrip": false,
42
  "single_word": false,
43
  "special": true
44
- },
45
- "32002": {
46
- "content": "<end_of_utterance>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
  }
53
  },
54
  "additional_special_tokens": [
55
  "<fake_token_around_image>",
56
- "<image>",
57
- "<end_of_utterance>"
58
  ],
59
  "bos_token": "<s>",
60
  "clean_up_tokenization_spaces": false,
 
41
  "rstrip": false,
42
  "single_word": false,
43
  "special": true
 
 
 
 
 
 
 
 
44
  }
45
  },
46
  "additional_special_tokens": [
47
  "<fake_token_around_image>",
48
+ "<image>"
 
49
  ],
50
  "bos_token": "<s>",
51
  "clean_up_tokenization_spaces": false,