Hugging Face's logo

pietrolesci
/
raw_tokenisers

Model card Files Community
pietrolesci commited on
Commit
72714a5
·
verified ·
1 Parent(s): d797a76

Upload folder using huggingface_hub

Browse files
Files changed (17) hide show
  1. .gitattributes +8 -0
  2. _bpe_finewebedu_2024-09-04T12-59-54/all_merges.jsonl +0 -0
  3. _bpe_finewebedu_2024-09-04T12-59-54/implemented_merges.jsonl +3 -0
  4. _bpe_finewebedu_2024-09-04T12-59-54/metadata.yaml +3 -0
  5. _bpe_finewebedu_2024-09-04T12-59-54/tokenizer.json +3 -0
  6. _bpe_minipile_2024-09-22T17-58-54/all_merges.jsonl +0 -0
  7. _bpe_minipile_2024-09-22T17-58-54/implemented_merges.jsonl +3 -0
  8. _bpe_minipile_2024-09-22T17-58-54/metadata.yaml +3 -0
  9. _bpe_minipile_2024-09-22T17-58-54/tokenizer.json +3 -0
  10. bpe_finewebedu_2024-10-11T20-50-21/all_merges.jsonl +0 -0
  11. bpe_finewebedu_2024-10-11T20-50-21/implemented_merges.jsonl +3 -0
  12. bpe_finewebedu_2024-10-11T20-50-21/metadata.yaml +3 -0
  13. bpe_finewebedu_2024-10-11T20-50-21/tokenizer.json +3 -0
  14. bpe_minipile_2024-10-11T11-23-27/all_merges.jsonl +0 -0
  15. bpe_minipile_2024-10-11T11-23-27/implemented_merges.jsonl +3 -0
  16. bpe_minipile_2024-10-11T11-23-27/metadata.yaml +3 -0
  17. bpe_minipile_2024-10-11T11-23-27/tokenizer.json +3 -0
.gitattributes CHANGED
@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ _bpe_finewebedu_2024-09-04T12-59-54/implemented_merges.jsonl filter=lfs diff=lfs merge=lfs -text
37
+ _bpe_finewebedu_2024-09-04T12-59-54/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ _bpe_minipile_2024-09-22T17-58-54/implemented_merges.jsonl filter=lfs diff=lfs merge=lfs -text
39
+ _bpe_minipile_2024-09-22T17-58-54/tokenizer.json filter=lfs diff=lfs merge=lfs -text
40
+ bpe_finewebedu_2024-10-11T20-50-21/implemented_merges.jsonl filter=lfs diff=lfs merge=lfs -text
41
+ bpe_finewebedu_2024-10-11T20-50-21/tokenizer.json filter=lfs diff=lfs merge=lfs -text
42
+ bpe_minipile_2024-10-11T11-23-27/implemented_merges.jsonl filter=lfs diff=lfs merge=lfs -text
43
+ bpe_minipile_2024-10-11T11-23-27/tokenizer.json filter=lfs diff=lfs merge=lfs -text
_bpe_finewebedu_2024-09-04T12-59-54/all_merges.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
_bpe_finewebedu_2024-09-04T12-59-54/implemented_merges.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:805bb4a844ee15c72279151e0780ab9460372977a76d027c52e9f06cde0f7359
3
+ size 35344669
_bpe_finewebedu_2024-09-04T12-59-54/metadata.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ max_vocab_size: 320000
2
+ num_docs: 9500000
3
+ eos_token: <|endoftext|>
_bpe_finewebedu_2024-09-04T12-59-54/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9650d2502332e2053ba5ae5b314325089f715e226b3d1503202429eaedc91cc0
3
+ size 23598133
_bpe_minipile_2024-09-22T17-58-54/all_merges.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
_bpe_minipile_2024-09-22T17-58-54/implemented_merges.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ace3f08cd83828046e11c288e4ee0beeb265898347a41b0b9521426a1a412da8
3
+ size 35047130
_bpe_minipile_2024-09-22T17-58-54/metadata.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ max_vocab_size: 320000
2
+ num_docs: 1000000
3
+ eos_token: <|endoftext|>
_bpe_minipile_2024-09-22T17-58-54/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90be5b2ce1a846fcf10b8e05263095075504f855a8a566a4efa39354141fdf83
3
+ size 23599177
bpe_finewebedu_2024-10-11T20-50-21/all_merges.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
bpe_finewebedu_2024-10-11T20-50-21/implemented_merges.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fe2934f2213f62c0a4640d5ce8fb608932c842afda7747d0d232a2e125df535
3
+ size 35464616
bpe_finewebedu_2024-10-11T20-50-21/metadata.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ max_vocab_size: 320000
2
+ num_docs: 20000000
3
+ eos_token: <|endoftext|>
bpe_finewebedu_2024-10-11T20-50-21/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32cf9af0b7730c3c70e70c710850ce5d2aa12c9f871d13078e4f0053c009b31c
3
+ size 23596285
bpe_minipile_2024-10-11T11-23-27/all_merges.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
bpe_minipile_2024-10-11T11-23-27/implemented_merges.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10c86b1954635fa7975e6f70b69c1bd3c996714dd775c2a0710d490b5e8404ac
3
+ size 35288764
bpe_minipile_2024-10-11T11-23-27/metadata.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ max_vocab_size: 320000
2
+ num_docs: 1000000
3
+ eos_token: <|endoftext|>
bpe_minipile_2024-10-11T11-23-27/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b597d8e1fa7b685bfda953a15e6981be9ea03e56f008c6cb521caf28d0a72d6
3
+ size 23777543