Hugging Face's logo

pietrolesci
/
tokenisers

Model card Files Community
pietrolesci commited on
Commit
521826d
·
verified ·
1 Parent(s): 6211c23

Upload folder using huggingface_hub

Browse files
Files changed (25) hide show
  1. _bpe32000finewebedu/raw_tok_path.txt +1 -0
  2. _bpe32000finewebedu/special_tokens_map.json +3 -0
  3. _bpe32000finewebedu/tokenizer.json +0 -0
  4. _bpe32000finewebedu/tokenizer_config.json +16 -0
  5. _bpe32000minipile/prefix_map_bpe32000minipile.jsonl +0 -0
  6. _bpe32000minipile/raw_tok_path.txt +1 -0
  7. _bpe32000minipile/special_tokens_map.json +3 -0
  8. _bpe32000minipile/tokenizer.json +0 -0
  9. _bpe32000minipile/tokenizer_config.json +16 -0
  10. bpe128000minipile/raw_tok_path.txt +1 -0
  11. bpe128000minipile/special_tokens_map.json +3 -0
  12. bpe128000minipile/tokenizer.json +0 -0
  13. bpe128000minipile/tokenizer_config.json +16 -0
  14. bpe32000finewebedu/raw_tok_path.txt +1 -0
  15. bpe32000finewebedu/special_tokens_map.json +3 -0
  16. bpe32000finewebedu/tokenizer.json +0 -0
  17. bpe32000finewebedu/tokenizer_config.json +16 -0
  18. bpe32000minipile/raw_tok_path.txt +1 -0
  19. bpe32000minipile/special_tokens_map.json +3 -0
  20. bpe32000minipile/tokenizer.json +0 -0
  21. bpe32000minipile/tokenizer_config.json +16 -0
  22. bpe8064minipile/raw_tok_path.txt +1 -0
  23. bpe8064minipile/special_tokens_map.json +3 -0
  24. bpe8064minipile/tokenizer.json +0 -0
  25. bpe8064minipile/tokenizer_config.json +16 -0
_bpe32000finewebedu/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_finewebedu_2024-09-04T12-59-54
_bpe32000finewebedu/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
_bpe32000finewebedu/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
_bpe32000finewebedu/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
_bpe32000minipile/prefix_map_bpe32000minipile.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
_bpe32000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-09-22T17-58-54
_bpe32000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
_bpe32000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
_bpe32000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
bpe128000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe128000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe128000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe128000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
bpe32000finewebedu/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_finewebedu_2024-10-11T20-50-21
bpe32000finewebedu/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe32000finewebedu/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe32000finewebedu/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
bpe32000minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe32000minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe32000minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe32000minipile/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }
bpe8064minipile/raw_tok_path.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ /home/pl487/rdd/outputs/tok_train/bpe_minipile_2024-10-11T11-23-27
bpe8064minipile/special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>"
3
+ }
bpe8064minipile/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
bpe8064minipile/tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "clean_up_tokenization_spaces": true,
13
+ "eos_token": "<|endoftext|>",
14
+ "model_max_length": 1000000000000000019884624838656,
15
+ "tokenizer_class": "PreTrainedTokenizerFast"
16
+ }