Upload tokenizer

tokenizer.json

@@ -2237,7 +2237,7 @@
       ],
       [
         "ce",
-        -7.480814933776856
+        -7.4808149337768555
       ],
       [
         "▁board",
@@ -2273,7 +2273,7 @@
       ],
       [
         "ive",
-        -7.513410091400147
+        -7.5134100914001465
       ],
       [
         "▁where",
@@ -2365,7 +2365,7 @@
       ],
       [
         "▁myself",
-        -7.600685596466064
+        -7.6006855964660645
       ],
       [
         "▁yen",
@@ -2433,7 +2433,7 @@
       ],
       [
         "▁hav",
-        -7.641457080841064
+        -7.6414570808410645
       ],
       [
         "▁As",
@@ -2549,7 +2549,7 @@
       ],
       [
         "That",
-        -7.752362728118896
+        -7.7523627281188965
       ],
       [
         "M",
@@ -2589,7 +2589,7 @@
       ],
       [
         "T",
-        -7.786305427551269
+        -7.7863054275512695
       ],
       [
         "▁read",
@@ -2693,11 +2693,11 @@
       ],
       [
         "ng",
-        -7.874188899993896
+        -7.8741888999938965
       ],
       [
         "▁mind",
-        -7.876381397247315
+        -7.8763813972473145
       ],
       [
         "▁cannot",
@@ -2713,7 +2713,7 @@
       ],
       [
         "▁hard",
-        -7.87647533416748
+        -7.8764753341674805
       ],
       [
         "▁nothing",
@@ -2789,7 +2789,7 @@
       ],
       [
         "The",
-        -7.928972721099853
+        -7.9289727210998535
       ],
       [
         "▁sp",
@@ -2857,7 +2857,7 @@
       ],
       [
         "▁put",
-        -7.977669239044189
+        -7.9776692390441895
       ],
       [
         "▁line",
@@ -5041,7 +5041,7 @@
       ],
       [
         "ian",
-        -9.022007942199709
+        -9.022007942199707
       ],
       [
         "rew",
@@ -5049,11 +5049,11 @@
       ],
       [
         "rate",
-        -9.027299880981444
+        -9.027299880981445
       ],
       [
         "▁fai",
-        -9.028965950012209
+        -9.028965950012207
       ],
       [
         "▁Hi",
@@ -5065,19 +5065,19 @@
       ],
       [
         "▁sw",
-        -9.040364265441896
+        -9.040364265441895
       ],
       [
         "ight",
-        -9.05210781097412
+        -9.052107810974121
       ],
       [
         "▁pass",
-        -9.059226036071776
+        -9.059226036071777
       ],
       [
         "▁accept",
-        -9.06157684326172
+        -9.061576843261719
       ],
       [
         "▁expect",
@@ -5085,7 +5085,7 @@
       ],
       [
         "▁Be",
-        -9.062178611755373
+        -9.062178611755371
       ],
       [
         "▁meet",
@@ -5185,11 +5185,11 @@
       ],
       [
         "▁sword",
-        -9.074662208557127
+        -9.074662208557129
       ],
       [
         "▁world",
-        -9.074662208557127
+        -9.074662208557129
       ],
       [
         "▁style",
@@ -5197,7 +5197,7 @@
       ],
       [
         "▁death",
-        -9.074671745300291
+        -9.074671745300293
       ],
       [
         "▁push",
@@ -5205,7 +5205,7 @@
       ],
       [
         "▁early",
-        -9.07468605041504
+        -9.074686050415039
       ],
       [
         "▁history",
@@ -5233,11 +5233,11 @@
       ],
       [
         "stead",
-        -9.074742317199709
+        -9.074742317199707
       ],
       [
         "▁added",
-        -9.074756622314451
+        -9.074756622314453
       ],
       [
         "▁stuff",
@@ -5249,7 +5249,7 @@
       ],
       [
         "▁temple",
-        -9.074835777282717
+        -9.074835777282715
       ],
       [
         "▁mention",
@@ -5273,11 +5273,11 @@
       ],
       [
         "▁post",
-        -9.075267791748049
+        -9.075267791748047
       ],
       [
         "▁catch",
-        -9.075643539428713
+        -9.075643539428711
       ],
       [
         "▁leg",
@@ -5297,11 +5297,11 @@
       ],
       [
         "▁arm",
-        -9.08041286468506
+        -9.080412864685059
       ],
       [
         "▁plan",
-        -9.081151008605955
+        -9.081151008605957
       ],
       [
         "▁pale",
@@ -5313,7 +5313,7 @@
       ],
       [
         "▁efforts",
-        -9.086438179016112
+        -9.086438179016113
       ],
       [
         "▁accepted",
@@ -5337,11 +5337,11 @@
       ],
       [
         "▁forty",
-        -9.09645652770996
+        -9.096456527709961
       ],
       [
         "▁walking",
-        -9.09681797027588
+        -9.096817970275879
       ],
       [
         "▁opened",
@@ -5349,7 +5349,7 @@
       ],
       [
         "Don",
-        -9.100522994995115
+        -9.100522994995117
       ],
       [
         "position",
@@ -5361,11 +5361,11 @@
       ],
       [
         "pri",
-        -9.103438377380373
+        -9.103438377380371
       ],
       [
         "▁forth",
-        -9.11118507385254
+        -9.111185073852539
       ],
       [
         "▁notice",
@@ -5373,7 +5373,7 @@
       ],
       [
         "▁play",
-        -9.123631477355955
+        -9.123631477355957
       ],
       [
         "raw",
@@ -5381,7 +5381,7 @@
       ],
       [
         "▁year",
-        -9.166281700134276
+        -9.166281700134277
       ],
       [
         "▁step",
@@ -5389,7 +5389,7 @@
       ],
       [
         "▁fin",
-        -9.175487518310549
+        -9.175487518310547
       ],
       [
         "▁direct",
@@ -5397,19 +5397,19 @@
       ],
       [
         "▁insist",
-        -9.185338020324709
+        -9.185338020324707
       ],
       [
         "▁enjoy",
-        -9.185760498046877
+        -9.185760498046875
       ],
       [
         "▁fashion",
-        -9.185760498046877
+        -9.185760498046875
       ],
       [
         "▁supper",
-        -9.185760498046877
+        -9.185760498046875
       ],
       [
         "Besides",
@@ -5493,11 +5493,11 @@
       ],
       [
         "▁trick",
-        -9.185766220092772
+        -9.185766220092773
       ],
       [
         "▁sharp",
-        -9.18577480316162
+        -9.185774803161621
       ],
       [
         "▁married",
@@ -5505,7 +5505,7 @@
       ],
       [
         "▁print",
-        -9.185812950134276
+        -9.185812950134277
       ],
       [
         "▁weak",
@@ -5513,7 +5513,7 @@
       ],
       [
         "▁wife",
-        -9.185831069946287
+        -9.185831069946289
       ],
       [
         "▁translation",
@@ -5529,7 +5529,7 @@
       ],
       [
         "▁below",
-        -9.185887336730955
+        -9.185887336730957
       ],
       [
         "▁smile",
@@ -5549,11 +5549,11 @@
       ],
       [
         "▁noise",
-        -9.186004638671877
+        -9.186004638671875
       ],
       [
         "▁intention",
-        -9.186009407043455
+        -9.186009407043457
       ],
       [
         "▁coward",
@@ -5573,7 +5573,7 @@
       ],
       [
         "▁loca",
-        -9.186867713928224
+        -9.186867713928223
       ],
       [
         "▁Ch",
@@ -5581,7 +5581,7 @@
       ],
       [
         "▁perform",
-        -9.188361167907717
+        -9.188361167907715
       ],
       [
         "ike",
@@ -5597,7 +5597,7 @@
       ],
       [
         "▁shot",
-        -9.188958168029783
+        -9.188958168029785
       ],
       [
         "▁display",
@@ -5605,7 +5605,7 @@
       ],
       [
         "machi",
-        -9.190868377685549
+        -9.190868377685547
       ],
       [
         "▁mental",
@@ -5613,7 +5613,7 @@
       ],
       [
         "▁sur",
-        -9.19241428375244
+        -9.192414283752441
       ],
       [
         "▁comply",
@@ -5629,11 +5629,11 @@
       ],
       [
         "▁distribute",
-        -9.193628311157228
+        -9.193628311157227
       ],
       [
         "▁sooner",
-        -9.193878173828123
+        -9.193878173828125
       ],
       [
         "▁provide",
@@ -5645,7 +5645,7 @@
       ],
       [
         "▁bor",
-        -9.203582763671877
+        -9.203582763671875
       ],
       [
         "▁standing",
@@ -5653,11 +5653,11 @@
       ],
       [
         "▁drawing",
-        -9.20570945739746
+        -9.205709457397461
       ],
       [
         "▁fast",
-        -9.211029052734377
+        -9.211029052734375
       ],
       [
         "▁cur",
@@ -5665,7 +5665,7 @@
       ],
       [
         "gen",
-        -9.213075637817385
+        -9.213075637817383
       ],
       [
         "etc",
@@ -5689,15 +5689,15 @@
       ],
       [
         "Pro",
-        -9.220240592956545
+        -9.220240592956543
       ],
       [
         "▁nose",
-        -9.221141815185549
+        -9.221141815185547
       ],
       [
         "▁Re",
-        -9.246124267578123
+        -9.246124267578125
       ],
       [
         "light",
@@ -5705,7 +5705,7 @@
       ],
       [
         "▁da",
-        -9.274735450744627
+        -9.274735450744629
       ],
       [
         "till",
@@ -5729,7 +5729,7 @@
       ],
       [
         "▁TH",
-        -9.305097579956056
+        -9.305097579956055
       ],
       [
         "becom",
@@ -5737,27 +5737,27 @@
       ],
       [
         "▁transfer",
-        -9.310699462890623
+        -9.310699462890625
       ],
       [
         "▁thir",
-        -9.310746192932127
+        -9.310746192932129
       ],
       [
         "sque",
-        -9.310760498046877
+        -9.310760498046875
       ],
       [
         "▁appeal",
-        -9.310760498046877
+        -9.310760498046875
       ],
       [
         "▁devil",
-        -9.310760498046877
+        -9.310760498046875
       ],
       [
         "▁judg",
-        -9.310760498046877
+        -9.310760498046875
       ],
       [
         "▁character",
@@ -5849,15 +5849,15 @@
       ],
       [
         "▁Russ",
-        -9.310765266418455
+        -9.310765266418457
       ],
       [
         "▁plain",
-        -9.310769081115724
+        -9.310769081115723
       ],
       [
         "▁glad",
-        -9.31077003479004
+        -9.310770034790039
       ],
       [
         "▁Section",
@@ -5869,7 +5869,7 @@
       ],
       [
         "▁deserv",
-        -9.310778617858888
+        -9.310778617858887
       ],
       [
         "▁credit",
@@ -5941,7 +5941,7 @@
       ],
       [
         "▁refus",
-        -9.311113357543944
+        -9.311113357543945
       ],
       [
         "▁drum",
@@ -5957,7 +5957,7 @@
       ],
       [
         "ordina",
-        -9.311413764953612
+        -9.311413764953613
       ],
       [
         "ware",
@@ -5977,7 +5977,7 @@
       ],
       [
         "running",
-        -9.31202793121338
+        -9.312027931213379
       ],
       [
         "▁mark",
@@ -6001,7 +6001,7 @@
       ],
       [
         "▁Ko",
-        -9.31403350830078
+        -9.314033508300781
       ],
       [
         "▁anybody",
@@ -6033,7 +6033,7 @@
       ],
       [
         "▁shoo",
-        -9.320408821105955
+        -9.320408821105957
       ],
       [
         "▁buy",
@@ -6041,7 +6041,7 @@
       ],
       [
         "dition",
-        -9.32142734527588
+        -9.321427345275879
       ],
       [
         "▁laws",
@@ -6113,7 +6113,7 @@
       ],
       [
         "IT",
-        -9.397068977355955
+        -9.397068977355957
       ],
       [
         "ward",
@@ -6125,7 +6125,7 @@
       ],
       [
         "▁simpl",
-        -9.40804958343506
+        -9.408049583435059
       ],
       [
         "uff",
@@ -6153,7 +6153,7 @@
       ],
       [
         "▁late",
-        -9.451498031616213
+        -9.451498031616211
       ],
       [
         "▁gentle",
@@ -6249,19 +6249,19 @@
       ],
       [
         "▁above",
-        -9.453619956970217
+        -9.453619956970215
       ],
       [
         "▁glance",
-        -9.453619956970217
+        -9.453619956970215
       ],
       [
         "▁shadow",
-        -9.453619956970217
+        -9.453619956970215
       ],
       [
         "▁unusual",
-        -9.453619956970217
+        -9.453619956970215
       ],
       [
         "▁gentlemen",
@@ -6297,7 +6297,7 @@
       ],
       [
         "▁repeat",
-        -9.453625679016112
+        -9.453625679016113
       ],
       [
         "▁decent",
@@ -6325,7 +6325,7 @@
       ],
       [
         "▁slip",
-        -9.453635215759276
+        -9.453635215759277
       ],
       [
         "doubt",
@@ -6337,7 +6337,7 @@
       ],
       [
         "▁instructor",
-        -9.453639030456545
+        -9.453639030456543
       ],
       [
         "▁education",
@@ -6357,11 +6357,11 @@
       ],
       [
         "▁demand",
-        -9.453653335571287
+        -9.453653335571289
       ],
       [
         "▁court",
-        -9.453690528869627
+        -9.453690528869629
       ],
       [
         "▁stuck",
@@ -6389,15 +6389,15 @@
       ],
       [
         "▁Con",
-        -9.454066276550291
+        -9.454066276550293
       ],
       [
         "▁boatman",
-        -9.454066276550291
+        -9.454066276550293
       ],
       [
         "▁type",
-        -9.45411777496338
+        -9.454117774963379
       ],
       [
         "promise",
@@ -6413,11 +6413,11 @@
       ],
       [
         "▁Up",
-        -9.454395294189451
+        -9.454395294189453
       ],
       [
         "▁yard",
-        -9.454404830932615
+        -9.454404830932617
       ],
       [
         "▁worth",
@@ -6437,7 +6437,7 @@
       ],
       [
         "SE",
-        -9.455034255981444
+        -9.455034255981445
       ],
       [
         "▁peep",
@@ -6449,7 +6449,7 @@
       ],
       [
         "▁subject",
-        -9.455371856689451
+        -9.455371856689453
       ],
       [
         "▁careful",
@@ -6457,7 +6457,7 @@
       ],
       [
         "▁nonsense",
-        -9.455403327941896
+        -9.455403327941895
       ],
       [
         "▁pine",
@@ -6469,15 +6469,15 @@
       ],
       [
         "▁ran",
-        -9.456183433532717
+        -9.456183433532715
       ],
       [
         "▁sometime",
-        -9.456221580505373
+        -9.456221580505371
       ],
       [
         "▁compar",
-        -9.456564903259276
+        -9.456564903259277
       ],
       [
         "▁Once",
@@ -6497,7 +6497,7 @@
       ],
       [
         "▁table",
-        -9.461024284362791
+        -9.461024284362793
       ],
       [
         "▁prove",
@@ -6525,11 +6525,11 @@
       ],
       [
         "paper",
-        -9.531177520751951
+        -9.531177520751953
       ],
       [
         "upon",
-        -9.53188133239746
+        -9.531881332397461
       ],
       [
         "some",
@@ -6537,7 +6537,7 @@
       ],
       [
         "▁favor",
-        -9.538238525390623
+        -9.538238525390625
       ],
       [
         "named",
@@ -6545,7 +6545,7 @@
       ],
       [
         "ode",
-        -9.582578659057615
+        -9.582578659057617
       ],
       [
         "cked",
@@ -6561,7 +6561,7 @@
       ],
       [
         "▁stat",
-        -9.59182834625244
+        -9.591828346252441
       ],
       [
         "▁effort",
@@ -6569,7 +6569,7 @@
       ],
       [
         "▁enter",
-        -9.609453201293944
+        -9.609453201293945
       ],
       [
         "▁surprise",
@@ -6581,11 +6581,11 @@
       ],
       [
         "▁rikisha",
-        -9.61831760406494
+        -9.618317604064941
       ],
       [
         "▁whi",
-        -9.61833953857422
+        -9.618339538574219
       ],
       [
         "ency",
@@ -6601,7 +6601,7 @@
       ],
       [
         "▁prefer",
-        -9.62006378173828
+        -9.620063781738281
       ],
       [
         "▁fun",
@@ -6609,15 +6609,15 @@
       ],
       [
         "▁require",
-        -9.620274543762209
+        -9.620274543762207
       ],
       [
         "▁imagin",
-        -9.620284080505373
+        -9.620284080505371
       ],
       [
         "▁vigor",
-        -9.620284080505373
+        -9.620284080505371
       ],
       [
         "Botchan",
@@ -6797,7 +6797,7 @@
       ],
       [
         "▁rattl",
-        -9.620288848876951
+        -9.620288848876953
       ],
       [
         "▁attention",
@@ -6805,7 +6805,7 @@
       ],
       [
         "▁New",
-        -9.62029266357422
+        -9.620292663574219
       ],
       [
         "claim",
@@ -6837,11 +6837,11 @@
       ],
       [
         "▁orange",
-        -9.620302200317385
+        -9.620302200317383
       ],
       [
         "▁dull",
-        -9.620311737060549
+        -9.620311737060547
       ],
       [
         "▁river",
@@ -6849,7 +6849,7 @@
       ],
       [
         "▁fuss",
-        -9.620326042175291
+        -9.620326042175293
       ],
       [
         "▁cap",
@@ -6885,7 +6885,7 @@
       ],
       [
         "nosu",
-        -9.62045192718506
+        -9.620451927185059
       ],
       [
         "▁flat",
@@ -6925,7 +6925,7 @@
       ],
       [
         "▁sauc",
-        -9.620758056640623
+        -9.620758056640625
       ],
       [
         "▁rice",
@@ -6949,11 +6949,11 @@
       ],
       [
         "▁treatment",
-        -9.621350288391112
+        -9.621350288391113
       ],
       [
         "▁favorit",
-        -9.621603965759276
+        -9.621603965759277
       ],
       [
         "▁glo",
@@ -6977,11 +6977,11 @@
       ],
       [
         "▁file",
-        -9.623125076293944
+        -9.623125076293945
       ],
       [
         "▁reasonable",
-        -9.623706817626951
+        -9.623706817626953
       ],
       [
         "▁anyone",
@@ -7009,11 +7009,11 @@
       ],
       [
         "▁hall",
-        -9.627115249633787
+        -9.627115249633789
       ],
       [
         "▁ice",
-        -9.63064670562744
+        -9.630646705627441
       ],
       [
         "ctual",
@@ -7041,7 +7041,7 @@
       ],
       [
         "site",
-        -9.645623207092283
+        -9.645623207092285
       ],
       [
         "you",
@@ -7061,7 +7061,7 @@
       ],
       [
         "duced",
-        -9.689046859741213
+        -9.689046859741211
       ],
       [
         "▁format",
@@ -7073,7 +7073,7 @@
       ],
       [
         "▁dar",
-        -9.69410800933838
+        -9.694108009338379
       ],
       [
         "pic",
@@ -7101,7 +7101,7 @@
       ],
       [
         "mats",
-        -9.743086814880373
+        -9.743086814880371
       ],
       [
         "knee",
@@ -7113,7 +7113,7 @@
       ],
       [
         "goo",
-        -9.794201850891112
+        -9.794201850891113
       ],
       [
         "▁necessar",
@@ -7125,7 +7125,7 @@
       ],
       [
         "▁law",
-        -9.801798820495604
+        -9.801798820495605
       ],
       [
         "▁decid",
@@ -7133,7 +7133,7 @@
       ],
       [
         "trict",
-        -9.808300018310549
+        -9.808300018310547
       ],
       [
         "▁settl",
@@ -7153,11 +7153,11 @@
       ],
       [
         "ould",
-        -9.819173812866213
+        -9.819173812866211
       ],
       [
         "pping",
-        -9.81940460205078
+        -9.819404602050781
       ],
       [
         "uck",
@@ -7165,7 +7165,7 @@
       ],
       [
         "awn",
-        -9.819780349731444
+        -9.819780349731445
       ],
       [
         "▁compli",
@@ -7177,7 +7177,7 @@
       ],
       [
         "eneral",
-        -9.820164680480955
+        -9.820164680480957
       ],
       [
         "mitted",
@@ -7385,51 +7385,51 @@
       ],
       [
         "approv",
-        -9.820286750793455
+        -9.820286750793457
       ],
       [
         "nounc",
-        -9.820286750793455
+        -9.820286750793457
       ],
       [
         "▁everywhere",
-        -9.820286750793455
+        -9.820286750793457
       ],
       [
         "▁snap",
-        -9.820286750793455
+        -9.820286750793457
       ],
       [
         "Suppos",
-        -9.820287704467772
+        -9.820287704467773
       ],
       [
         "▁begun",
-        -9.820287704467772
+        -9.820287704467773
       ],
       [
         "▁fix",
-        -9.820287704467772
+        -9.820287704467773
       ],
       [
         "▁respectfully",
-        -9.820287704467772
+        -9.820287704467773
       ],
       [
         "▁result",
-        -9.820287704467772
+        -9.820287704467773
       ],
       [
         "▁yellow",
-        -9.820287704467772
+        -9.820287704467773
       ],
       [
         "umbling",
-        -9.820290565490724
+        -9.820290565490723
       ],
       [
         "▁official",
-        -9.820290565490724
+        -9.820290565490723
       ],
       [
         "▁group",
@@ -7449,15 +7449,15 @@
       ],
       [
         "▁firm",
-        -9.820300102233888
+        -9.820300102233887
       ],
       [
         "▁gramma",
-        -9.820300102233888
+        -9.820300102233887
       ],
       [
         "▁panel",
-        -9.820300102233888
+        -9.820300102233887
       ],
       [
         "▁itself",
@@ -7501,7 +7501,7 @@
       ],
       [
         "▁slap",
-        -9.82034397125244
+        -9.820343971252441
       ],
       [
         "▁noisy",
@@ -7517,11 +7517,11 @@
       ],
       [
         "chief",
-        -9.820390701293944
+        -9.820390701293945
       ],
       [
         "▁shouted",
-        -9.820408821105955
+        -9.820408821105957
       ],
       [
         "▁ball",
@@ -7541,7 +7541,7 @@
       ],
       [
         "▁edge",
-        -9.820578575134276
+        -9.820578575134277
       ],
       [
         "▁shak",
@@ -7549,7 +7549,7 @@
       ],
       [
         "▁pride",
-        -9.820841789245604
+        -9.820841789245605
       ],
       [
         "▁expenses",
@@ -7561,11 +7561,11 @@
       ],
       [
         "▁Tak",
-        -9.820967674255373
+        -9.820967674255371
       ],
       [
         "▁broke",
-        -9.821123123168944
+        -9.821123123168945
       ],
       [
         "struck",
@@ -7609,11 +7609,11 @@
       ],
       [
         "▁tight",
-        -9.821817398071287
+        -9.821817398071289
       ],
       [
         "▁Every",
-        -9.821944236755373
+        -9.821944236755371
       ],
       [
         "reafter",
@@ -7621,7 +7621,7 @@
       ],
       [
         "▁brain",
-        -9.822479248046877
+        -9.822479248046875
       ],
       [
         "▁manage",
@@ -7633,7 +7633,7 @@
       ],
       [
         "grace",
-        -9.829718589782717
+        -9.829718589782715
       ],
       [
         "formed",
@@ -7669,7 +7669,7 @@
       ],
       [
         "empt",
-        -9.88385772705078
+        -9.883857727050781
       ],
       [
         "dog",
@@ -7677,7 +7677,7 @@
       ],
       [
         "Here",
-        -9.904093742370604
+        -9.904093742370605
       ],
       [
         "▁prais",
@@ -7697,7 +7697,7 @@
       ],
       [
         "face",
-        -9.97946071624756
+        -9.979460716247559
       ],
       [
         "▁punish",
@@ -7733,7 +7733,7 @@
       ],
       [
         "aka",
-        -10.034581184387209
+        -10.034581184387207
       ],
       [
         "pecul",
@@ -7745,7 +7745,7 @@
       ],
       [
         "▁advis",
-        -10.064568519592283
+        -10.064568519592285
       ],
       [
         "educat",
@@ -7757,7 +7757,7 @@
       ],
       [
         "▁respon",
-        -10.070267677307127
+        -10.070267677307129
       ],
       [
         "▁express",
@@ -8117,35 +8117,35 @@
       ],
       [
         "version",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "▁engage",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "▁hobby",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "▁length",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "▁logic",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "▁midnight",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "▁polish",
-        -10.070286750793455
+        -10.070286750793457
       ],
       [
         "(",
-        -10.312033653259276
+        -10.312033653259277
       ],
       [
         "*",
@@ -8157,7 +8157,7 @@
       ],
       [
         "q",
-        -11.903417587280272
+        -11.903417587280273
       ],
       [
         "j",
@@ -8165,7 +8165,7 @@
       ],
       [
         "v",
-        -11.90361785888672
+        -11.903617858886719
       ]
     ],
     "byte_fallback": false

tokenizer_config.json

@@ -74,15 +74,11 @@
   "extra_ids": 0,
   "legacy": true,
   "mask_token": "<MASK>",
-  "max_length": 2048,
   "model_max_length": 2048,
   "pad_token": "<pad>",
   "sep_token": "<SEP>",
   "sp_model_kwargs": {},
   "split_special_tokens": true,
-  "stride": 0,
   "tokenizer_class": "T5Tokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }