add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"{": 32100, "}": 32101, "<c>": 32102, "</c>": 32103, "<size>": 32104, "<Aa>": 32105, "<Ab>": 32106, "<Ac>": 32107, "<Ad>": 32108, "<Ae>": 32109, "<Af>": 32110, "<Ag>": 32111, "<Ah>": 32112, "<Ai>": 32113, "<Aj>": 32114, "<Ak>": 32115, "<Al>": 32116, "<Am>": 32117, "<An>": 32118, "<Ao>": 32119, "<Ap>": 32120, "<Aq>": 32121, "<Ar>": 32122, "<As>": 32123, "<At>": 32124, "<Au>": 32125, "<Av>": 32126, "<Aw>": 32127, "<Ax>": 32128, "<Ay>": 32129, "<Ba>": 32130, "<Bb>": 32131, "<Bc>": 32132, "<Bd>": 32133, "<Be>": 32134, "<Bf>": 32135, "<Bg>": 32136, "<Bh>": 32137, "<Bi>": 32138, "<Bj>": 32139, "<Bk>": 32140, "<Bl>": 32141, "<Bm>": 32142, "<Bn>": 32143, "<Bo>": 32144, "<Bp>": 32145, "<Bq>": 32146, "<Br>": 32147, "<Bs>": 32148, "<Bt>": 32149, "<Bu>": 32150, "<Bv>": 32151, "<Bw>": 32152, "<Bx>": 32153, "<By>": 32154, "<Ca>": 32155, "<Cb>": 32156, "<Cc>": 32157, "<Cd>": 32158, "<Ce>": 32159, "<Cf>": 32160, "<Cg>": 32161, "<Ch>": 32162, "<Ci>": 32163, "<Cj>": 32164, "<Ck>": 32165, "<Cl>": 32166, "<Cm>": 32167, "<Cn>": 32168, "<Co>": 32169, "<Cp>": 32170, "<Cq>": 32171, "<Cr>": 32172, "<Cs>": 32173, "<Ct>": 32174, "<Cu>": 32175, "<Cv>": 32176, "<Cw>": 32177, "<Cx>": 32178, "<Cy>": 32179, "<Da>": 32180, "<Db>": 32181, "<Dc>": 32182, "<Dd>": 32183, "<De>": 32184, "<Df>": 32185, "<Dg>": 32186, "<Dh>": 32187, "<Di>": 32188, "<Dj>": 32189, "<Dk>": 32190, "<Dl>": 32191, "<Dm>": 32192, "<Dn>": 32193, "<Do>": 32194, "<Dp>": 32195, "<Dq>": 32196, "<Dr>": 32197, "<Ds>": 32198, "<Dt>": 32199, "<Du>": 32200, "<Dv>": 32201, "<Dw>": 32202, "<Dx>": 32203, "<Dy>": 32204, "<Ea>": 32205, "<Eb>": 32206, "<Ec>": 32207, "<Ed>": 32208, "<Ee>": 32209, "<Ef>": 32210, "<Eg>": 32211, "<Eh>": 32212, "<Ei>": 32213, "<Ej>": 32214, "<Ek>": 32215, "<El>": 32216, "<Em>": 32217, "<En>": 32218, "<Eo>": 32219, "<Ep>": 32220, "<Eq>": 32221, "<Er>": 32222, "<Es>": 32223, "<Et>": 32224, "<Eu>": 32225, "<Ev>": 32226, "<Ew>": 32227, "<Ex>": 32228, "<Ey>": 32229, "<Fa>": 32230, "<Fb>": 32231, "<Fc>": 32232, "<Fd>": 32233, "<Fe>": 32234, "<Ff>": 32235, "<Fg>": 32236, "<Fh>": 32237, "<Fi>": 32238, "<Fj>": 32239, "<Fk>": 32240, "<Fl>": 32241, "<Fm>": 32242, "<Fn>": 32243, "<Fo>": 32244, "<Fp>": 32245, "<Fq>": 32246, "<Fr>": 32247, "<Fs>": 32248, "<Ft>": 32249, "<Fu>": 32250, "<Fv>": 32251, "<Fw>": 32252, "<Fx>": 32253, "<Fy>": 32254, "<Ga>": 32255, "<Gb>": 32256, "<Gc>": 32257, "<Gd>": 32258, "<Ge>": 32259, "<Gf>": 32260, "<Gg>": 32261, "<Gh>": 32262, "<Gi>": 32263, "<Gj>": 32264, "<Gk>": 32265, "<Gl>": 32266, "<Gm>": 32267, "<Gn>": 32268, "<Go>": 32269, "<Gp>": 32270, "<Gq>": 32271, "<Gr>": 32272, "<Gs>": 32273, "<Gt>": 32274, "<Gu>": 32275, "<Gv>": 32276, "<Gw>": 32277, "<Gx>": 32278, "<Gy>": 32279, "<Ha>": 32280, "<Hb>": 32281, "<Hc>": 32282, "<Hd>": 32283, "<He>": 32284, "<Hf>": 32285, "<Hg>": 32286, "<Hh>": 32287, "<Hi>": 32288, "<Hj>": 32289, "<Hk>": 32290, "<Hl>": 32291, "<Hm>": 32292, "<Hn>": 32293, "<Ho>": 32294, "<Hp>": 32295, "<Hq>": 32296, "<Hr>": 32297, "<Hs>": 32298, "<Ht>": 32299, "<Hu>": 32300, "<Hv>": 32301, "<Hw>": 32302, "<Hx>": 32303, "<Hy>": 32304, "<Ia>": 32305, "<Ib>": 32306, "<Ic>": 32307, "<Id>": 32308, "<Ie>": 32309, "<If>": 32310, "<Ig>": 32311, "<Ih>": 32312, "<Ii>": 32313, "<Ij>": 32314, "<Ik>": 32315, "<Il>": 32316, "<Im>": 32317, "<In>": 32318, "<Io>": 32319, "<Ip>": 32320, "<Iq>": 32321, "<Ir>": 32322, "<Is>": 32323, "<It>": 32324, "<Iu>": 32325, "<Iv>": 32326, "<Iw>": 32327, "<Ix>": 32328, "<Iy>": 32329, "<Ja>": 32330, "<Jb>": 32331, "<Jc>": 32332, "<Jd>": 32333, "<Je>": 32334, "<Jf>": 32335, "<Jg>": 32336, "<Jh>": 32337, "<Ji>": 32338, "<Jj>": 32339, "<Jk>": 32340, "<Jl>": 32341, "<Jm>": 32342, "<Jn>": 32343, "<Jo>": 32344, "<Jp>": 32345, "<Jq>": 32346, "<Jr>": 32347, "<Js>": 32348, "<Jt>": 32349, "<Ju>": 32350, "<Jv>": 32351, "<Jw>": 32352, "<Jx>": 32353, "<Jy>": 32354, "<Ka>": 32355, "<Kb>": 32356, "<Kc>": 32357, "<Kd>": 32358, "<Ke>": 32359, "<Kf>": 32360, "<Kg>": 32361, "<Kh>": 32362, "<Ki>": 32363, "<Kj>": 32364, "<Kk>": 32365, "<Kl>": 32366, "<Km>": 32367, "<Kn>": 32368, "<Ko>": 32369, "<Kp>": 32370, "<Kq>": 32371, "<Kr>": 32372, "<Ks>": 32373, "<Kt>": 32374, "<Ku>": 32375, "<Kv>": 32376, "<Kw>": 32377, "<Kx>": 32378, "<Ky>": 32379, "<La>": 32380, "<Lb>": 32381, "<Lc>": 32382, "<Ld>": 32383, "<Le>": 32384, "<Lf>": 32385, "<Lg>": 32386, "<Lh>": 32387, "<Li>": 32388, "<Lj>": 32389, "<Lk>": 32390, "<Ll>": 32391, "<Lm>": 32392, "<Ln>": 32393, "<Lo>": 32394, "<Lp>": 32395, "<Lq>": 32396, "<Lr>": 32397, "<Ls>": 32398, "<Lt>": 32399, "<Lu>": 32400, "<Lv>": 32401, "<Lw>": 32402, "<Lx>": 32403, "<Ly>": 32404, "<Ma>": 32405, "<Mb>": 32406, "<Mc>": 32407, "<Md>": 32408, "<Me>": 32409, "<Mf>": 32410, "<Mg>": 32411, "<Mh>": 32412, "<Mi>": 32413, "<Mj>": 32414, "<Mk>": 32415, "<Ml>": 32416, "<Mm>": 32417, "<Mn>": 32418, "<Mo>": 32419, "<Mp>": 32420, "<Mq>": 32421, "<Mr>": 32422, "<Ms>": 32423, "<Mt>": 32424, "<Mu>": 32425, "<Mv>": 32426, "<Mw>": 32427, "<Mx>": 32428, "<My>": 32429, "<Na>": 32430, "<Nb>": 32431, "<Nc>": 32432, "<Nd>": 32433, "<Ne>": 32434, "<Nf>": 32435, "<Ng>": 32436, "<Nh>": 32437, "<Ni>": 32438, "<Nj>": 32439, "<Nk>": 32440, "<Nl>": 32441, "<Nm>": 32442, "<Nn>": 32443, "<No>": 32444, "<Np>": 32445, "<Nq>": 32446, "<Nr>": 32447, "<Ns>": 32448, "<Nt>": 32449, "<Nu>": 32450, "<Nv>": 32451, "<Nw>": 32452, "<Nx>": 32453, "<Ny>": 32454, "<Oa>": 32455, "<Ob>": 32456, "<Oc>": 32457, "<Od>": 32458, "<Oe>": 32459, "<Of>": 32460, "<Og>": 32461, "<Oh>": 32462, "<Oi>": 32463, "<Oj>": 32464, "<Ok>": 32465, "<Ol>": 32466, "<Om>": 32467, "<On>": 32468, "<Oo>": 32469, "<Op>": 32470, "<Oq>": 32471, "<Or>": 32472, "<Os>": 32473, "<Ot>": 32474, "<Ou>": 32475, "<Ov>": 32476, "<Ow>": 32477, "<Ox>": 32478, "<Oy>": 32479, "<Pa>": 32480, "<Pb>": 32481, "<Pc>": 32482, "<Pd>": 32483, "<Pe>": 32484, "<Pf>": 32485, "<Pg>": 32486, "<Ph>": 32487, "<Pi>": 32488, "<Pj>": 32489, "<Pk>": 32490, "<Pl>": 32491, "<Pm>": 32492, "<Pn>": 32493, "<Po>": 32494, "<Pp>": 32495, "<Pq>": 32496, "<Pr>": 32497, "<Ps>": 32498, "<Pt>": 32499, "<Pu>": 32500, "<Pv>": 32501, "<Pw>": 32502, "<Px>": 32503, "<Py>": 32504, "<Qa>": 32505, "<Qb>": 32506, "<Qc>": 32507, "<Qd>": 32508, "<Qe>": 32509, "<Qf>": 32510, "<Qg>": 32511, "<Qh>": 32512, "<Qi>": 32513, "<Qj>": 32514, "<Qk>": 32515, "<Ql>": 32516, "<Qm>": 32517, "<Qn>": 32518, "<Qo>": 32519, "<Qp>": 32520, "<Qq>": 32521, "<Qr>": 32522, "<Qs>": 32523, "<Qt>": 32524, "<Qu>": 32525, "<Qv>": 32526, "<Qw>": 32527, "<Qx>": 32528, "<Qy>": 32529, "<Ra>": 32530, "<Rb>": 32531, "<Rc>": 32532, "<Rd>": 32533, "<Re>": 32534, "<Rf>": 32535, "<Rg>": 32536, "<Rh>": 32537, "<Ri>": 32538, "<Rj>": 32539, "<Rk>": 32540, "<Rl>": 32541, "<Rm>": 32542, "<Rn>": 32543, "<Ro>": 32544, "<Rp>": 32545, "<Rq>": 32546, "<Rr>": 32547, "<Rs>": 32548, "<Rt>": 32549, "<Ru>": 32550, "<Rv>": 32551, "<Rw>": 32552, "<Rx>": 32553, "<Ry>": 32554, "<Sa>": 32555, "<Sb>": 32556, "<Sc>": 32557, "<Sd>": 32558, "<Se>": 32559, "<Sf>": 32560, "<Sg>": 32561, "<Sh>": 32562, "<Si>": 32563, "<Sj>": 32564, "<Sk>": 32565, "<Sl>": 32566, "<Sm>": 32567, "<Sn>": 32568, "<So>": 32569, "<Sp>": 32570, "<Sq>": 32571, "<Sr>": 32572, "<Ss>": 32573, "<St>": 32574, "<Su>": 32575, "<Sv>": 32576, "<Sw>": 32577, "<Sx>": 32578, "<Sy>": 32579, "<Ta>": 32580, "<Tb>": 32581, "<Tc>": 32582, "<Td>": 32583, "<Te>": 32584, "<Tf>": 32585, "<Tg>": 32586, "<Th>": 32587, "<Ti>": 32588, "<Tj>": 32589, "<Tk>": 32590, "<Tl>": 32591, "<Tm>": 32592, "<Tn>": 32593, "<To>": 32594, "<Tp>": 32595, "<Tq>": 32596, "<Tr>": 32597, "<Ts>": 32598, "<Tt>": 32599, "<Tu>": 32600, "<Tv>": 32601, "<Tw>": 32602, "<Tx>": 32603, "<Ty>": 32604, "<Ua>": 32605, "<Ub>": 32606, "<Uc>": 32607, "<Ud>": 32608, "<Ue>": 32609, "<Uf>": 32610, "<Ug>": 32611, "<Uh>": 32612, "<Ui>": 32613, "<Uj>": 32614, "<Uk>": 32615, "<Ul>": 32616, "<Um>": 32617, "<Un>": 32618, "<Uo>": 32619, "<Up>": 32620, "<Uq>": 32621, "<Ur>": 32622, "<Us>": 32623, "<Ut>": 32624, "<Uu>": 32625, "<Uv>": 32626, "<Uw>": 32627, "<Ux>": 32628, "<Uy>": 32629, "<Va>": 32630, "<Vb>": 32631, "<Vc>": 32632, "<Vd>": 32633, "<Ve>": 32634, "<Vf>": 32635, "<Vg>": 32636, "<Vh>": 32637, "<Vi>": 32638, "<Vj>": 32639, "<Vk>": 32640, "<Vl>": 32641, "<Vm>": 32642, "<Vn>": 32643, "<Vo>": 32644, "<Vp>": 32645, "<Vq>": 32646, "<Vr>": 32647, "<Vs>": 32648, "<Vt>": 32649, "<Vu>": 32650, "<Vv>": 32651, "<Vw>": 32652, "<Vx>": 32653, "<Vy>": 32654, "<Wa>": 32655, "<Wb>": 32656, "<Wc>": 32657, "<Wd>": 32658, "<We>": 32659, "<Wf>": 32660, "<Wg>": 32661, "<Wh>": 32662, "<Wi>": 32663, "<Wj>": 32664, "<Wk>": 32665, "<Wl>": 32666, "<Wm>": 32667, "<Wn>": 32668, "<Wo>": 32669, "<Wp>": 32670, "<Wq>": 32671, "<Wr>": 32672, "<Ws>": 32673, "<Wt>": 32674, "<Wu>": 32675, "<Wv>": 32676, "<Ww>": 32677, "<Wx>": 32678, "<Wy>": 32679, "<Xa>": 32680, "<Xb>": 32681, "<Xc>": 32682, "<Xd>": 32683, "<Xe>": 32684, "<Xf>": 32685, "<Xg>": 32686, "<Xh>": 32687, "<Xi>": 32688, "<Xj>": 32689, "<Xk>": 32690, "<Xl>": 32691, "<Xm>": 32692, "<Xn>": 32693, "<Xo>": 32694, "<Xp>": 32695, "<Xq>": 32696, "<Xr>": 32697, "<Xs>": 32698, "<Xt>": 32699, "<Xu>": 32700, "<Xv>": 32701, "<Xw>": 32702, "<Xx>": 32703, "<Xy>": 32704, "<Ya>": 32705, "<Yb>": 32706, "<Yc>": 32707, "<Yd>": 32708, "<Ye>": 32709, "<Yf>": 32710, "<Yg>": 32711, "<Yh>": 32712, "<Yi>": 32713, "<Yj>": 32714, "<Yk>": 32715, "<Yl>": 32716, "<Ym>": 32717, "<Yn>": 32718, "<Yo>": 32719, "<Yp>": 32720, "<Yq>": 32721, "<Yr>": 32722, "<Ys>": 32723, "<Yt>": 32724, "<Yu>": 32725, "<Yv>": 32726, "<Yw>": 32727, "<Yx>": 32728, "<Yy>": 32729, "<Za>": 32730, "<Zb>": 32731, "<Zc>": 32732, "<Zd>": 32733, "<Ze>": 32734, "<Zf>": 32735, "<Zg>": 32736, "<Zh>": 32737, "<Zi>": 32738, "<Zj>": 32739, "<Zk>": 32740, "<Zl>": 32741, "<Zm>": 32742, "<Zn>": 32743, "<Zo>": 32744, "<Zp>": 32745, "<Zq>": 32746, "<Zr>": 32747, "<Zs>": 32748, "<Zt>": 32749, "<Zu>": 32750, "<Zv>": 32751, "<Zw>": 32752, "<Zx>": 32753, "<Zy>": 32754, "<mask>": 32755}
+{"{": 32100, "}": 32101, "<c>": 32102, "</c>": 32103, "<size>": 32104, "<t>": 32105, "</t>": 32106, "<a>": 32107, "</a>": 32108, "<mask>": 32109}

special_tokens_map.json

@@ -1 +1 @@
-{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>", "additional_special_tokens": ["<Aa>", "<Ab>", "<Ac>", "<Ad>", "<Ae>", "<Af>", "<Ag>", "<Ah>", "<Ai>", "<Aj>", "<Ak>", "<Al>", "<Am>", "<An>", "<Ao>", "<Ap>", "<Aq>", "<Ar>", "<As>", "<At>", "<Au>", "<Av>", "<Aw>", "<Ax>", "<Ay>", "<Ba>", "<Bb>", "<Bc>", "<Bd>", "<Be>", "<Bf>", "<Bg>", "<Bh>", "<Bi>", "<Bj>", "<Bk>", "<Bl>", "<Bm>", "<Bn>", "<Bo>", "<Bp>", "<Bq>", "<Br>", "<Bs>", "<Bt>", "<Bu>", "<Bv>", "<Bw>", "<Bx>", "<By>", "<Ca>", "<Cb>", "<Cc>", "<Cd>", "<Ce>", "<Cf>", "<Cg>", "<Ch>", "<Ci>", "<Cj>", "<Ck>", "<Cl>", "<Cm>", "<Cn>", "<Co>", "<Cp>", "<Cq>", "<Cr>", "<Cs>", "<Ct>", "<Cu>", "<Cv>", "<Cw>", "<Cx>", "<Cy>", "<Da>", "<Db>", "<Dc>", "<Dd>", "<De>", "<Df>", "<Dg>", "<Dh>", "<Di>", "<Dj>", "<Dk>", "<Dl>", "<Dm>", "<Dn>", "<Do>", "<Dp>", "<Dq>", "<Dr>", "<Ds>", "<Dt>", "<Du>", "<Dv>", "<Dw>", "<Dx>", "<Dy>", "<Ea>", "<Eb>", "<Ec>", "<Ed>", "<Ee>", "<Ef>", "<Eg>", "<Eh>", "<Ei>", "<Ej>", "<Ek>", "<El>", "<Em>", "<En>", "<Eo>", "<Ep>", "<Eq>", "<Er>", "<Es>", "<Et>", "<Eu>", "<Ev>", "<Ew>", "<Ex>", "<Ey>", "<Fa>", "<Fb>", "<Fc>", "<Fd>", "<Fe>", "<Ff>", "<Fg>", "<Fh>", "<Fi>", "<Fj>", "<Fk>", "<Fl>", "<Fm>", "<Fn>", "<Fo>", "<Fp>", "<Fq>", "<Fr>", "<Fs>", "<Ft>", "<Fu>", "<Fv>", "<Fw>", "<Fx>", "<Fy>", "<Ga>", "<Gb>", "<Gc>", "<Gd>", "<Ge>", "<Gf>", "<Gg>", "<Gh>", "<Gi>", "<Gj>", "<Gk>", "<Gl>", "<Gm>", "<Gn>", "<Go>", "<Gp>", "<Gq>", "<Gr>", "<Gs>", "<Gt>", "<Gu>", "<Gv>", "<Gw>", "<Gx>", "<Gy>", "<Ha>", "<Hb>", "<Hc>", "<Hd>", "<He>", "<Hf>", "<Hg>", "<Hh>", "<Hi>", "<Hj>", "<Hk>", "<Hl>", "<Hm>", "<Hn>", "<Ho>", "<Hp>", "<Hq>", "<Hr>", "<Hs>", "<Ht>", "<Hu>", "<Hv>", "<Hw>", "<Hx>", "<Hy>", "<Ia>", "<Ib>", "<Ic>", "<Id>", "<Ie>", "<If>", "<Ig>", "<Ih>", "<Ii>", "<Ij>", "<Ik>", "<Il>", "<Im>", "<In>", "<Io>", "<Ip>", "<Iq>", "<Ir>", "<Is>", "<It>", "<Iu>", "<Iv>", "<Iw>", "<Ix>", "<Iy>", "<Ja>", "<Jb>", "<Jc>", "<Jd>", "<Je>", "<Jf>", "<Jg>", "<Jh>", "<Ji>", "<Jj>", "<Jk>", "<Jl>", "<Jm>", "<Jn>", "<Jo>", "<Jp>", "<Jq>", "<Jr>", "<Js>", "<Jt>", "<Ju>", "<Jv>", "<Jw>", "<Jx>", "<Jy>", "<Ka>", "<Kb>", "<Kc>", "<Kd>", "<Ke>", "<Kf>", "<Kg>", "<Kh>", "<Ki>", "<Kj>", "<Kk>", "<Kl>", "<Km>", "<Kn>", "<Ko>", "<Kp>", "<Kq>", "<Kr>", "<Ks>", "<Kt>", "<Ku>", "<Kv>", "<Kw>", "<Kx>", "<Ky>", "<La>", "<Lb>", "<Lc>", "<Ld>", "<Le>", "<Lf>", "<Lg>", "<Lh>", "<Li>", "<Lj>", "<Lk>", "<Ll>", "<Lm>", "<Ln>", "<Lo>", "<Lp>", "<Lq>", "<Lr>", "<Ls>", "<Lt>", "<Lu>", "<Lv>", "<Lw>", "<Lx>", "<Ly>", "<Ma>", "<Mb>", "<Mc>", "<Md>", "<Me>", "<Mf>", "<Mg>", "<Mh>", "<Mi>", "<Mj>", "<Mk>", "<Ml>", "<Mm>", "<Mn>", "<Mo>", "<Mp>", "<Mq>", "<Mr>", "<Ms>", "<Mt>", "<Mu>", "<Mv>", "<Mw>", "<Mx>", "<My>", "<Na>", "<Nb>", "<Nc>", "<Nd>", "<Ne>", "<Nf>", "<Ng>", "<Nh>", "<Ni>", "<Nj>", "<Nk>", "<Nl>", "<Nm>", "<Nn>", "<No>", "<Np>", "<Nq>", "<Nr>", "<Ns>", "<Nt>", "<Nu>", "<Nv>", "<Nw>", "<Nx>", "<Ny>", "<Oa>", "<Ob>", "<Oc>", "<Od>", "<Oe>", "<Of>", "<Og>", "<Oh>", "<Oi>", "<Oj>", "<Ok>", "<Ol>", "<Om>", "<On>", "<Oo>", "<Op>", "<Oq>", "<Or>", "<Os>", "<Ot>", "<Ou>", "<Ov>", "<Ow>", "<Ox>", "<Oy>", "<Pa>", "<Pb>", "<Pc>", "<Pd>", "<Pe>", "<Pf>", "<Pg>", "<Ph>", "<Pi>", "<Pj>", "<Pk>", "<Pl>", "<Pm>", "<Pn>", "<Po>", "<Pp>", "<Pq>", "<Pr>", "<Ps>", "<Pt>", "<Pu>", "<Pv>", "<Pw>", "<Px>", "<Py>", "<Qa>", "<Qb>", "<Qc>", "<Qd>", "<Qe>", "<Qf>", "<Qg>", "<Qh>", "<Qi>", "<Qj>", "<Qk>", "<Ql>", "<Qm>", "<Qn>", "<Qo>", "<Qp>", "<Qq>", "<Qr>", "<Qs>", "<Qt>", "<Qu>", "<Qv>", "<Qw>", "<Qx>", "<Qy>", "<Ra>", "<Rb>", "<Rc>", "<Rd>", "<Re>", "<Rf>", "<Rg>", "<Rh>", "<Ri>", "<Rj>", "<Rk>", "<Rl>", "<Rm>", "<Rn>", "<Ro>", "<Rp>", "<Rq>", "<Rr>", "<Rs>", "<Rt>", "<Ru>", "<Rv>", "<Rw>", "<Rx>", "<Ry>", "<Sa>", "<Sb>", "<Sc>", "<Sd>", "<Se>", "<Sf>", "<Sg>", "<Sh>", "<Si>", "<Sj>", "<Sk>", "<Sl>", "<Sm>", "<Sn>", "<So>", "<Sp>", "<Sq>", "<Sr>", "<Ss>", "<St>", "<Su>", "<Sv>", "<Sw>", "<Sx>", "<Sy>", "<Ta>", "<Tb>", "<Tc>", "<Td>", "<Te>", "<Tf>", "<Tg>", "<Th>", "<Ti>", "<Tj>", "<Tk>", "<Tl>", "<Tm>", "<Tn>", "<To>", "<Tp>", "<Tq>", "<Tr>", "<Ts>", "<Tt>", "<Tu>", "<Tv>", "<Tw>", "<Tx>", "<Ty>", "<Ua>", "<Ub>", "<Uc>", "<Ud>", "<Ue>", "<Uf>", "<Ug>", "<Uh>", "<Ui>", "<Uj>", "<Uk>", "<Ul>", "<Um>", "<Un>", "<Uo>", "<Up>", "<Uq>", "<Ur>", "<Us>", "<Ut>", "<Uu>", "<Uv>", "<Uw>", "<Ux>", "<Uy>", "<Va>", "<Vb>", "<Vc>", "<Vd>", "<Ve>", "<Vf>", "<Vg>", "<Vh>", "<Vi>", "<Vj>", "<Vk>", "<Vl>", "<Vm>", "<Vn>", "<Vo>", "<Vp>", "<Vq>", "<Vr>", "<Vs>", "<Vt>", "<Vu>", "<Vv>", "<Vw>", "<Vx>", "<Vy>", "<Wa>", "<Wb>", "<Wc>", "<Wd>", "<We>", "<Wf>", "<Wg>", "<Wh>", "<Wi>", "<Wj>", "<Wk>", "<Wl>", "<Wm>", "<Wn>", "<Wo>", "<Wp>", "<Wq>", "<Wr>", "<Ws>", "<Wt>", "<Wu>", "<Wv>", "<Ww>", "<Wx>", "<Wy>", "<Xa>", "<Xb>", "<Xc>", "<Xd>", "<Xe>", "<Xf>", "<Xg>", "<Xh>", "<Xi>", "<Xj>", "<Xk>", "<Xl>", "<Xm>", "<Xn>", "<Xo>", "<Xp>", "<Xq>", "<Xr>", "<Xs>", "<Xt>", "<Xu>", "<Xv>", "<Xw>", "<Xx>", "<Xy>", "<Ya>", "<Yb>", "<Yc>", "<Yd>", "<Ye>", "<Yf>", "<Yg>", "<Yh>", "<Yi>", "<Yj>", "<Yk>", "<Yl>", "<Ym>", "<Yn>", "<Yo>", "<Yp>", "<Yq>", "<Yr>", "<Ys>", "<Yt>", "<Yu>", "<Yv>", "<Yw>", "<Yx>", "<Yy>", "<Za>", "<Zb>", "<Zc>", "<Zd>", "<Ze>", "<Zf>", "<Zg>", "<Zh>", "<Zi>", "<Zj>", "<Zk>", "<Zl>", "<Zm>", "<Zn>", "<Zo>", "<Zp>", "<Zq>", "<Zr>", "<Zs>", "<Zt>", "<Zu>", "<Zv>", "<Zw>", "<Zx>", "<Zy>"]}
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>", "additional_special_tokens": ["<t>", "</t>", "<a>", "</a>"]}

tokenizer_config.json

@@ -1 +1 @@
-{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"], "special_tokens_map_file": null, "tokenizer_file": "/root/.cache/huggingface/transformers/61fb045860657f2da69b3ce49169c420069767c86f25c2f3716abc702f09ee85.8627f1bd5d270a9fd2e5a51c8bec3223896587cc3cfe13edeabb0992ab43c529", "name_or_path": "razent/SciFive-base-Pubmed_PMC"}
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 100, "additional_special_tokens": ["<extra_id_0>", "<extra_id_1>", "<extra_id_2>", "<extra_id_3>", "<extra_id_4>", "<extra_id_5>", "<extra_id_6>", "<extra_id_7>", "<extra_id_8>", "<extra_id_9>", "<extra_id_10>", "<extra_id_11>", "<extra_id_12>", "<extra_id_13>", "<extra_id_14>", "<extra_id_15>", "<extra_id_16>", "<extra_id_17>", "<extra_id_18>", "<extra_id_19>", "<extra_id_20>", "<extra_id_21>", "<extra_id_22>", "<extra_id_23>", "<extra_id_24>", "<extra_id_25>", "<extra_id_26>", "<extra_id_27>", "<extra_id_28>", "<extra_id_29>", "<extra_id_30>", "<extra_id_31>", "<extra_id_32>", "<extra_id_33>", "<extra_id_34>", "<extra_id_35>", "<extra_id_36>", "<extra_id_37>", "<extra_id_38>", "<extra_id_39>", "<extra_id_40>", "<extra_id_41>", "<extra_id_42>", "<extra_id_43>", "<extra_id_44>", "<extra_id_45>", "<extra_id_46>", "<extra_id_47>", "<extra_id_48>", "<extra_id_49>", "<extra_id_50>", "<extra_id_51>", "<extra_id_52>", "<extra_id_53>", "<extra_id_54>", "<extra_id_55>", "<extra_id_56>", "<extra_id_57>", "<extra_id_58>", "<extra_id_59>", "<extra_id_60>", "<extra_id_61>", "<extra_id_62>", "<extra_id_63>", "<extra_id_64>", "<extra_id_65>", "<extra_id_66>", "<extra_id_67>", "<extra_id_68>", "<extra_id_69>", "<extra_id_70>", "<extra_id_71>", "<extra_id_72>", "<extra_id_73>", "<extra_id_74>", "<extra_id_75>", "<extra_id_76>", "<extra_id_77>", "<extra_id_78>", "<extra_id_79>", "<extra_id_80>", "<extra_id_81>", "<extra_id_82>", "<extra_id_83>", "<extra_id_84>", "<extra_id_85>", "<extra_id_86>", "<extra_id_87>", "<extra_id_88>", "<extra_id_89>", "<extra_id_90>", "<extra_id_91>", "<extra_id_92>", "<extra_id_93>", "<extra_id_94>", "<extra_id_95>", "<extra_id_96>", "<extra_id_97>", "<extra_id_98>", "<extra_id_99>"], "sp_model_kwargs": {}, "model_max_length": 512, "name_or_path": "ozcangundes/T5-base-for-BioQA", "special_tokens_map_file": "/home/sarm/.cache/huggingface/transformers/60cb6b54d1ae19573814b37af7a3b160c8b8cc6ca12db8b9681ff665691900ee.c94798918c92ded6aeef2d2f0e666d2cc4145eca1aa6e1336fde07f2e13e2f46", "tokenizer_class": "T5Tokenizer"}