| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.73125161607048, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7150765613453124, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7443000568532008, | |
| 6.40691279840157e-265 | |
| ], | |
| "spearman": [ | |
| 0.759890982808152, | |
| 1.6901750172343795e-282 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7216210553290073, | |
| 3.539646733532031e-222 | |
| ], | |
| "spearman": [ | |
| 0.7304939228930519, | |
| 2.4834600870262812e-230 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7322719424112719, | |
| "mean": 0.7323909094175627, | |
| "wmean": 0.7319808833295888 | |
| }, | |
| "spearman": { | |
| "all": 0.7295849518142924, | |
| "mean": 0.7351538223488387, | |
| "wmean": 0.7253317970625808 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7596859741218331, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7018231072465838, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.777647576791967, | |
| 1.9353259230163057e-102 | |
| ], | |
| "spearman": [ | |
| 0.7337127308506983, | |
| 1.2717377929862408e-85 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7566490759566877, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6936198984184592, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7589921709943909, | |
| "mean": 0.7646608756234959, | |
| "wmean": 0.7590833755598704 | |
| }, | |
| "spearman": { | |
| "all": 0.6992260214830822, | |
| "mean": 0.7097185788385804, | |
| "wmean": 0.6993578712141356 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 81.81, | |
| "acc": 81.41, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.87, | |
| "acc": 87.23, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.31, | |
| "acc": 95.0, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 89.07, | |
| "acc": 89.01, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 84.4, | |
| "acc": 86.27, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 80.43, | |
| "acc": 87.6, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 70.39, | |
| "acc": 67.07, | |
| "f1": 74.82, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.44512999156378674, | |
| 8.846234223022077e-38 | |
| ], | |
| "spearman": [ | |
| 0.4509834995822136, | |
| 7.501496598491026e-39 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8158138179672533, | |
| 4.485078256906633e-180 | |
| ], | |
| "spearman": [ | |
| 0.8238805942224864, | |
| 1.2395324040896922e-186 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.5025777230618875, | |
| 9.478558829456811e-31 | |
| ], | |
| "spearman": [ | |
| 0.6028335206096967, | |
| 9.378736061336642e-47 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7075050481901065, | |
| 7.017744961203796e-115 | |
| ], | |
| "spearman": [ | |
| 0.6550259208484078, | |
| 4.101580378925782e-93 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6149709443544354, | |
| 7.125268059032375e-43 | |
| ], | |
| "spearman": [ | |
| 0.6179122177365513, | |
| 2.2209746905791215e-43 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.5765393545095476, | |
| "mean": 0.617199505027494, | |
| "wmean": 0.6281831483184317 | |
| }, | |
| "spearman": { | |
| "all": 0.5812709943392725, | |
| "mean": 0.6301271505998711, | |
| "wmean": 0.6340621209223184 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5404400970895422, | |
| 1.0024601828505787e-15 | |
| ], | |
| "spearman": [ | |
| 0.5700597864027528, | |
| 1.1134900500394056e-17 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7696057345254828, | |
| 7.053396385656569e-148 | |
| ], | |
| "spearman": [ | |
| 0.7771493600170609, | |
| 1.3424669074971778e-152 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.770711585062639, | |
| 1.6777951910420866e-111 | |
| ], | |
| "spearman": [ | |
| 0.7682009295399416, | |
| 2.3759249362431374e-110 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7077401152320973, | |
| "mean": 0.6935858055592213, | |
| "wmean": 0.7411444523094507 | |
| }, | |
| "spearman": { | |
| "all": 0.7340343966035406, | |
| "mean": 0.705136691986585, | |
| "wmean": 0.7477093607432154 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5050347882312676, | |
| 1.6792481128963397e-30 | |
| ], | |
| "spearman": [ | |
| 0.5006511549237668, | |
| 6.351186067873574e-30 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7505543283782676, | |
| 1.480995559109095e-55 | |
| ], | |
| "spearman": [ | |
| 0.7172316918319939, | |
| 1.1485152196025994e-48 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7381665393126864, | |
| 5.392499213485816e-130 | |
| ], | |
| "spearman": [ | |
| 0.6987173657276973, | |
| 6.630592581022267e-111 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8184796221575482, | |
| 3.315905577355896e-182 | |
| ], | |
| "spearman": [ | |
| 0.7800399098279688, | |
| 1.8607695353393358e-154 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8151737890413434, | |
| 1.4397424024788037e-179 | |
| ], | |
| "spearman": [ | |
| 0.8163228192853189, | |
| 1.7682187396229704e-180 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.6920222067407532, | |
| 5.678306228875661e-108 | |
| ], | |
| "spearman": [ | |
| 0.6388729139230673, | |
| 2.906642337192504e-87 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6768936191011258, | |
| "mean": 0.719905212310311, | |
| "wmean": 0.7334169523084798 | |
| }, | |
| "spearman": { | |
| "all": 0.6371047443411619, | |
| "mean": 0.6919726425866354, | |
| "wmean": 0.704247275690222 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.6715484368711376, | |
| 1.6630650795173534e-50 | |
| ], | |
| "spearman": [ | |
| 0.6791768037991774, | |
| 4.803053709588311e-52 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.6752231441950838, | |
| 5.897966571800042e-101 | |
| ], | |
| "spearman": [ | |
| 0.6804903297262569, | |
| 4.1825635033521844e-103 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.7171089394500281, | |
| 1.8875842940680164e-60 | |
| ], | |
| "spearman": [ | |
| 0.7340838023228742, | |
| 1.1234078020923912e-64 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7957787507836831, | |
| 4.2857031841813906e-165 | |
| ], | |
| "spearman": [ | |
| 0.7959923201180171, | |
| 3.028810020400643e-165 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.847852285173722, | |
| 2.736037311458429e-208 | |
| ], | |
| "spearman": [ | |
| 0.8592562196355571, | |
| 5.9692336633848975e-220 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7528553221562304, | |
| "mean": 0.7415023112947309, | |
| "wmean": 0.753295717078268 | |
| }, | |
| "spearman": { | |
| "all": 0.7672378740860561, | |
| "mean": 0.7497998951203765, | |
| "wmean": 0.7605922931352141 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.6932025545095016, | |
| 1.0441476948753323e-37 | |
| ], | |
| "spearman": [ | |
| 0.716984850240451, | |
| 2.2505666854089843e-41 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7647140202671684, | |
| 4.751336314867901e-49 | |
| ], | |
| "spearman": [ | |
| 0.7735811184492498, | |
| 7.591019148208835e-51 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8094434845489986, | |
| 1.2478658754392033e-54 | |
| ], | |
| "spearman": [ | |
| 0.8231197837843675, | |
| 5.9498794720031536e-58 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.7946011230206756, | |
| 2.306149833128848e-54 | |
| ], | |
| "spearman": [ | |
| 0.8178716303963535, | |
| 5.115171930640644e-60 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7803307090745555, | |
| 4.4645254931736274e-44 | |
| ], | |
| "spearman": [ | |
| 0.7927716723925775, | |
| 2.1646221593031153e-46 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7636230650001257, | |
| "mean": 0.7684583782841801, | |
| "wmean": 0.7669738900099788 | |
| }, | |
| "spearman": { | |
| "all": 0.7833717936510003, | |
| "mean": 0.7848658110525999, | |
| "wmean": 0.7835610101788375 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.5812709943392725, | |
| "eval_senteval-STS13": 0.7340343966035406, | |
| "eval_senteval-STS14": 0.6371047443411619, | |
| "eval_senteval-STS15": 0.7672378740860561, | |
| "eval_senteval-STS16": 0.7833717936510003, | |
| "eval_senteval-STSBenchmark": 0.7295849518142924, | |
| "eval_senteval-SICKRelatedness": 0.6992260214830822, | |
| "eval_senteval-avg_sts_7": 0.7045472537597723, | |
| "eval_senteval-MR": 81.81, | |
| "eval_senteval-CR": 87.87, | |
| "eval_senteval-SUBJ": 95.31, | |
| "eval_senteval-MPQA": 89.07, | |
| "eval_senteval-SST2": 84.4, | |
| "eval_senteval-TREC": 80.43, | |
| "eval_senteval-MRPC": 70.39, | |
| "eval_senteval-avg_transfer": 84.18285714285716 | |
| } | |