|
---
|
|
model-index:
|
|
- name: stella_en_1.5B_v5
|
|
results:
|
|
- dataset:
|
|
config: en
|
|
name: MTEB AmazonCounterfactualClassification (en)
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
|
|
split: test
|
|
type: mteb/amazon_counterfactual
|
|
metrics:
|
|
- type: accuracy
|
|
value: 92.86567164179104
|
|
- type: ap
|
|
value: 72.13503907102613
|
|
- type: ap_weighted
|
|
value: 72.13503907102613
|
|
- type: f1
|
|
value: 89.5586886376355
|
|
- type: f1_weighted
|
|
value: 93.13621183004571
|
|
- type: main_score
|
|
value: 92.86567164179104
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB AmazonPolarityClassification
|
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
|
|
split: test
|
|
type: mteb/amazon_polarity
|
|
metrics:
|
|
- type: accuracy
|
|
value: 97.16485
|
|
- type: ap
|
|
value: 96.05546315415225
|
|
- type: ap_weighted
|
|
value: 96.05546315415225
|
|
- type: f1
|
|
value: 97.16351087403213
|
|
- type: f1_weighted
|
|
value: 97.16351087403213
|
|
- type: main_score
|
|
value: 97.16485
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: en
|
|
name: MTEB AmazonReviewsClassification (en)
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
|
|
split: test
|
|
type: mteb/amazon_reviews_multi
|
|
metrics:
|
|
- type: accuracy
|
|
value: 59.358
|
|
- type: f1
|
|
value: 59.0264615883114
|
|
- type: f1_weighted
|
|
value: 59.0264615883114
|
|
- type: main_score
|
|
value: 59.358
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB ArguAna
|
|
revision: c22ab2a51041ffd869aaddef7af8d8215647e41a
|
|
split: test
|
|
type: mteb/arguana
|
|
metrics:
|
|
- type: main_score
|
|
value: 65.269
|
|
- type: map_at_1
|
|
value: 41.607
|
|
- type: map_at_10
|
|
value: 57.104
|
|
- type: map_at_100
|
|
value: 57.621
|
|
- type: map_at_1000
|
|
value: 57.621
|
|
- type: map_at_20
|
|
value: 57.533
|
|
- type: map_at_3
|
|
value: 52.891999999999996
|
|
- type: map_at_5
|
|
value: 55.371
|
|
- type: mrr_at_1
|
|
value: 42.318634423897585
|
|
- type: mrr_at_10
|
|
value: 57.353970511865406
|
|
- type: mrr_at_100
|
|
value: 57.88398078476526
|
|
- type: mrr_at_1000
|
|
value: 57.88467807648422
|
|
- type: mrr_at_20
|
|
value: 57.796730533206166
|
|
- type: mrr_at_3
|
|
value: 53.200568990042775
|
|
- type: mrr_at_5
|
|
value: 55.6330014224753
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 24.54414600428287
|
|
- type: nauc_map_at_1000_max
|
|
value: -8.389738078358459
|
|
- type: nauc_map_at_1000_std
|
|
value: -18.188787645801366
|
|
- type: nauc_map_at_100_diff1
|
|
value: 24.543138576462308
|
|
- type: nauc_map_at_100_max
|
|
value: -8.390896839752044
|
|
- type: nauc_map_at_100_std
|
|
value: -18.192549240185247
|
|
- type: nauc_map_at_10_diff1
|
|
value: 24.219607088995822
|
|
- type: nauc_map_at_10_max
|
|
value: -8.245734391254308
|
|
- type: nauc_map_at_10_std
|
|
value: -18.229706566466447
|
|
- type: nauc_map_at_1_diff1
|
|
value: 29.325201664812788
|
|
- type: nauc_map_at_1_max
|
|
value: -11.742800494823971
|
|
- type: nauc_map_at_1_std
|
|
value: -18.610215769702528
|
|
- type: nauc_map_at_20_diff1
|
|
value: 24.471097562798803
|
|
- type: nauc_map_at_20_max
|
|
value: -8.318035874000799
|
|
- type: nauc_map_at_20_std
|
|
value: -18.171541096773108
|
|
- type: nauc_map_at_3_diff1
|
|
value: 24.275846107642824
|
|
- type: nauc_map_at_3_max
|
|
value: -8.212242049581894
|
|
- type: nauc_map_at_3_std
|
|
value: -17.920379368937496
|
|
- type: nauc_map_at_5_diff1
|
|
value: 23.873692493209255
|
|
- type: nauc_map_at_5_max
|
|
value: -8.110347163828767
|
|
- type: nauc_map_at_5_std
|
|
value: -18.20863325596931
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 22.656410956419975
|
|
- type: nauc_mrr_at_1000_max
|
|
value: -8.924888102233243
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -18.103674384502526
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 22.655448817140968
|
|
- type: nauc_mrr_at_100_max
|
|
value: -8.926034318499038
|
|
- type: nauc_mrr_at_100_std
|
|
value: -18.10743930104164
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 22.297536272996872
|
|
- type: nauc_mrr_at_10_max
|
|
value: -8.836407556658274
|
|
- type: nauc_mrr_at_10_std
|
|
value: -18.1598393044477
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 27.419572424489708
|
|
- type: nauc_mrr_at_1_max
|
|
value: -11.42241314820691
|
|
- type: nauc_mrr_at_1_std
|
|
value: -18.54893865856313
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 22.590227214657418
|
|
- type: nauc_mrr_at_20_max
|
|
value: -8.849986456376993
|
|
- type: nauc_mrr_at_20_std
|
|
value: -18.0862391777352
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 22.415270167774988
|
|
- type: nauc_mrr_at_3_max
|
|
value: -8.692871854156435
|
|
- type: nauc_mrr_at_3_std
|
|
value: -17.6740102891955
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 21.96284578521464
|
|
- type: nauc_mrr_at_5_max
|
|
value: -8.757031535546025
|
|
- type: nauc_mrr_at_5_std
|
|
value: -18.210766964081294
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 23.939400161569115
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: -7.866999120512983
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -17.981457019643617
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 23.920033349619317
|
|
- type: nauc_ndcg_at_100_max
|
|
value: -7.889849409678031
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -18.054931990360537
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 22.543020461303534
|
|
- type: nauc_ndcg_at_10_max
|
|
value: -7.072111788010867
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -18.26397604573537
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 29.325201664812788
|
|
- type: nauc_ndcg_at_1_max
|
|
value: -11.742800494823971
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -18.610215769702528
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 23.551587021207972
|
|
- type: nauc_ndcg_at_20_max
|
|
value: -7.298056222649139
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -18.056004880930608
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 22.669089506345273
|
|
- type: nauc_ndcg_at_3_max
|
|
value: -7.278024373570137
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -17.816657759914193
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 21.72619728226575
|
|
- type: nauc_ndcg_at_5_max
|
|
value: -6.959741647471228
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -18.35173705190235
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: 5.0388241058076995
|
|
- type: nauc_precision_at_1000_max
|
|
value: 34.439879624882145
|
|
- type: nauc_precision_at_1000_std
|
|
value: 77.22610895194498
|
|
- type: nauc_precision_at_100_diff1
|
|
value: 1.340670767252794
|
|
- type: nauc_precision_at_100_max
|
|
value: 19.30870025961241
|
|
- type: nauc_precision_at_100_std
|
|
value: 35.37688289157788
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 7.734227153124332
|
|
- type: nauc_precision_at_10_max
|
|
value: 4.202399088422237
|
|
- type: nauc_precision_at_10_std
|
|
value: -18.383890254046698
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 29.325201664812788
|
|
- type: nauc_precision_at_1_max
|
|
value: -11.742800494823971
|
|
- type: nauc_precision_at_1_std
|
|
value: -18.610215769702528
|
|
- type: nauc_precision_at_20_diff1
|
|
value: 9.48070999361637
|
|
- type: nauc_precision_at_20_max
|
|
value: 19.056709637253025
|
|
- type: nauc_precision_at_20_std
|
|
value: -13.266821166159485
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 17.245260303409747
|
|
- type: nauc_precision_at_3_max
|
|
value: -4.202455033452335
|
|
- type: nauc_precision_at_3_std
|
|
value: -17.514264039955332
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 12.074628162049974
|
|
- type: nauc_precision_at_5_max
|
|
value: -1.9145501461107832
|
|
- type: nauc_precision_at_5_std
|
|
value: -19.162525528916344
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 5.038824105805915
|
|
- type: nauc_recall_at_1000_max
|
|
value: 34.43987962487738
|
|
- type: nauc_recall_at_1000_std
|
|
value: 77.22610895193765
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 1.3406707672497025
|
|
- type: nauc_recall_at_100_max
|
|
value: 19.30870025960776
|
|
- type: nauc_recall_at_100_std
|
|
value: 35.37688289157515
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 7.734227153124366
|
|
- type: nauc_recall_at_10_max
|
|
value: 4.202399088421976
|
|
- type: nauc_recall_at_10_std
|
|
value: -18.38389025404673
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 29.325201664812788
|
|
- type: nauc_recall_at_1_max
|
|
value: -11.742800494823971
|
|
- type: nauc_recall_at_1_std
|
|
value: -18.610215769702528
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 9.480709993616845
|
|
- type: nauc_recall_at_20_max
|
|
value: 19.05670963725301
|
|
- type: nauc_recall_at_20_std
|
|
value: -13.266821166158651
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 17.24526030340978
|
|
- type: nauc_recall_at_3_max
|
|
value: -4.202455033452323
|
|
- type: nauc_recall_at_3_std
|
|
value: -17.51426403995538
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 12.074628162049992
|
|
- type: nauc_recall_at_5_max
|
|
value: -1.914550146110865
|
|
- type: nauc_recall_at_5_std
|
|
value: -19.162525528916362
|
|
- type: ndcg_at_1
|
|
value: 41.607
|
|
- type: ndcg_at_10
|
|
value: 65.269
|
|
- type: ndcg_at_100
|
|
value: 67.289
|
|
- type: ndcg_at_1000
|
|
value: 67.29899999999999
|
|
- type: ndcg_at_20
|
|
value: 66.76299999999999
|
|
- type: ndcg_at_3
|
|
value: 56.604
|
|
- type: ndcg_at_5
|
|
value: 61.07900000000001
|
|
- type: precision_at_1
|
|
value: 41.607
|
|
- type: precision_at_10
|
|
value: 9.118
|
|
- type: precision_at_100
|
|
value: 0.996
|
|
- type: precision_at_1000
|
|
value: 0.1
|
|
- type: precision_at_20
|
|
value: 4.8469999999999995
|
|
- type: precision_at_3
|
|
value: 22.451
|
|
- type: precision_at_5
|
|
value: 15.647
|
|
- type: recall_at_1
|
|
value: 41.607
|
|
- type: recall_at_10
|
|
value: 91.181
|
|
- type: recall_at_100
|
|
value: 99.57300000000001
|
|
- type: recall_at_1000
|
|
value: 99.644
|
|
- type: recall_at_20
|
|
value: 96.942
|
|
- type: recall_at_3
|
|
value: 67.354
|
|
- type: recall_at_5
|
|
value: 78.236
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB ArxivClusteringP2P
|
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
|
|
split: test
|
|
type: mteb/arxiv-clustering-p2p
|
|
metrics:
|
|
- type: main_score
|
|
value: 55.437138353189994
|
|
- type: v_measure
|
|
value: 55.437138353189994
|
|
- type: v_measure_std
|
|
value: 14.718556601335491
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB ArxivClusteringS2S
|
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
|
|
split: test
|
|
type: mteb/arxiv-clustering-s2s
|
|
metrics:
|
|
- type: main_score
|
|
value: 50.65858459544658
|
|
- type: v_measure
|
|
value: 50.65858459544658
|
|
- type: v_measure_std
|
|
value: 14.887033747525146
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB AskUbuntuDupQuestions
|
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
|
|
split: test
|
|
type: mteb/askubuntudupquestions-reranking
|
|
metrics:
|
|
- type: main_score
|
|
value: 67.32597152838535
|
|
- type: map
|
|
value: 67.32597152838535
|
|
- type: mrr
|
|
value: 78.98683111286988
|
|
- type: nAUC_map_diff1
|
|
value: 16.8624639710487
|
|
- type: nAUC_map_max
|
|
value: 24.91996491142433
|
|
- type: nAUC_map_std
|
|
value: 17.91865808793225
|
|
- type: nAUC_mrr_diff1
|
|
value: 25.03766425631947
|
|
- type: nAUC_mrr_max
|
|
value: 41.64561939958336
|
|
- type: nAUC_mrr_std
|
|
value: 23.179909345891968
|
|
task:
|
|
type: Reranking
|
|
- dataset:
|
|
config: default
|
|
name: MTEB BIOSSES
|
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
|
|
split: test
|
|
type: mteb/biosses-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 85.790820496042
|
|
- type: cosine_spearman
|
|
value: 83.10731534330517
|
|
- type: euclidean_pearson
|
|
value: 84.61741304343133
|
|
- type: euclidean_spearman
|
|
value: 83.17297949010973
|
|
- type: main_score
|
|
value: 83.10731534330517
|
|
- type: manhattan_pearson
|
|
value: 85.2137696526676
|
|
- type: manhattan_spearman
|
|
value: 84.39168195786738
|
|
- type: pearson
|
|
value: 85.790820496042
|
|
- type: spearman
|
|
value: 83.10731534330517
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB Banking77Classification
|
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
|
|
split: test
|
|
type: mteb/banking77
|
|
metrics:
|
|
- type: accuracy
|
|
value: 89.78896103896105
|
|
- type: f1
|
|
value: 89.76107366333488
|
|
- type: f1_weighted
|
|
value: 89.76107366333488
|
|
- type: main_score
|
|
value: 89.78896103896105
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB BiorxivClusteringP2P
|
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
|
|
split: test
|
|
type: mteb/biorxiv-clustering-p2p
|
|
metrics:
|
|
- type: main_score
|
|
value: 50.68092296236376
|
|
- type: v_measure
|
|
value: 50.68092296236376
|
|
- type: v_measure_std
|
|
value: 0.7832640983085436
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB BiorxivClusteringS2S
|
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
|
|
split: test
|
|
type: mteb/biorxiv-clustering-s2s
|
|
metrics:
|
|
- type: main_score
|
|
value: 46.86629236732983
|
|
- type: v_measure
|
|
value: 46.86629236732983
|
|
- type: v_measure_std
|
|
value: 0.8784322236350974
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB CQADupstackRetrieval
|
|
revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4
|
|
split: test
|
|
type: mteb/cqadupstack
|
|
metrics:
|
|
- type: main_score
|
|
value: 47.74883333333334
|
|
- type: map_at_1
|
|
value: 30.179249999999996
|
|
- type: map_at_10
|
|
value: 41.60824999999999
|
|
- type: map_at_100
|
|
value: 42.94008333333332
|
|
- type: map_at_1000
|
|
value: 43.04666666666667
|
|
- type: map_at_20
|
|
value: 42.36833333333334
|
|
- type: map_at_3
|
|
value: 38.23491666666666
|
|
- type: map_at_5
|
|
value: 40.10183333333333
|
|
- type: mrr_at_1
|
|
value: 36.47676085808166
|
|
- type: mrr_at_10
|
|
value: 46.300991916437155
|
|
- type: mrr_at_100
|
|
value: 47.12155753713262
|
|
- type: mrr_at_1000
|
|
value: 47.168033610799945
|
|
- type: mrr_at_20
|
|
value: 46.80405724560391
|
|
- type: mrr_at_3
|
|
value: 43.77000352801797
|
|
- type: mrr_at_5
|
|
value: 45.22295361704542
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 46.953671666941524
|
|
- type: nauc_map_at_1000_max
|
|
value: 32.260396316089675
|
|
- type: nauc_map_at_1000_std
|
|
value: 0.6657766120094878
|
|
- type: nauc_map_at_100_diff1
|
|
value: 46.94717463394555
|
|
- type: nauc_map_at_100_max
|
|
value: 32.25088350678177
|
|
- type: nauc_map_at_100_std
|
|
value: 0.6257017014549283
|
|
- type: nauc_map_at_10_diff1
|
|
value: 46.974678429336464
|
|
- type: nauc_map_at_10_max
|
|
value: 31.862230807295504
|
|
- type: nauc_map_at_10_std
|
|
value: -0.14758828549579284
|
|
- type: nauc_map_at_1_diff1
|
|
value: 52.48913346466124
|
|
- type: nauc_map_at_1_max
|
|
value: 29.874374024967725
|
|
- type: nauc_map_at_1_std
|
|
value: -2.433547569836134
|
|
- type: nauc_map_at_20_diff1
|
|
value: 46.96088684217651
|
|
- type: nauc_map_at_20_max
|
|
value: 32.08954208613205
|
|
- type: nauc_map_at_20_std
|
|
value: 0.25946321113436527
|
|
- type: nauc_map_at_3_diff1
|
|
value: 47.703230121518345
|
|
- type: nauc_map_at_3_max
|
|
value: 30.977880095983107
|
|
- type: nauc_map_at_3_std
|
|
value: -1.342777563991804
|
|
- type: nauc_map_at_5_diff1
|
|
value: 47.1615010199957
|
|
- type: nauc_map_at_5_max
|
|
value: 31.420885812683284
|
|
- type: nauc_map_at_5_std
|
|
value: -0.8789297099444306
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 46.69178645962615
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 34.392807413340655
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 1.6155464863667934
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 46.67417236349189
|
|
- type: nauc_mrr_at_100_max
|
|
value: 34.384607045512624
|
|
- type: nauc_mrr_at_100_std
|
|
value: 1.6259917384109652
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 46.60497560446239
|
|
- type: nauc_mrr_at_10_max
|
|
value: 34.32918897817958
|
|
- type: nauc_mrr_at_10_std
|
|
value: 1.39387793769014
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 51.61608573254137
|
|
- type: nauc_mrr_at_1_max
|
|
value: 35.18105023234596
|
|
- type: nauc_mrr_at_1_std
|
|
value: 0.17943702145478177
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 46.635943069860254
|
|
- type: nauc_mrr_at_20_max
|
|
value: 34.37050973118794
|
|
- type: nauc_mrr_at_20_std
|
|
value: 1.5346464678860607
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 47.154389369038334
|
|
- type: nauc_mrr_at_3_max
|
|
value: 34.41036411855465
|
|
- type: nauc_mrr_at_3_std
|
|
value: 0.924551812357872
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 46.6690101691763
|
|
- type: nauc_mrr_at_5_max
|
|
value: 34.29740388138466
|
|
- type: nauc_mrr_at_5_std
|
|
value: 1.0567184149139792
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 45.375448289173264
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 33.47957083714482
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 3.192251100225568
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 44.93601014699499
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 33.21249888295249
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 3.609842852934217
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 44.87893284011915
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 32.384885249478515
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 1.454493065035396
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 51.61608573254137
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 35.18105023234596
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 0.17943702145478177
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 44.867752179050605
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 32.689535921840196
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 2.337765158573901
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 45.87485821381341
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 32.33282450558947
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 0.0681643829273283
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 45.202902131892394
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 32.1026971523917
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 0.3565572833774486
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -8.935267931198956
|
|
- type: nauc_precision_at_1000_max
|
|
value: 6.464981960169269
|
|
- type: nauc_precision_at_1000_std
|
|
value: 10.662786182234633
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -1.64091517847155
|
|
- type: nauc_precision_at_100_max
|
|
value: 15.175617871025024
|
|
- type: nauc_precision_at_100_std
|
|
value: 16.924256989248075
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 15.676651966277047
|
|
- type: nauc_precision_at_10_max
|
|
value: 26.243734188847117
|
|
- type: nauc_precision_at_10_std
|
|
value: 10.601741034956333
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 51.61608573254137
|
|
- type: nauc_precision_at_1_max
|
|
value: 35.18105023234596
|
|
- type: nauc_precision_at_1_std
|
|
value: 0.17943702145478177
|
|
- type: nauc_precision_at_20_diff1
|
|
value: 9.447267260198654
|
|
- type: nauc_precision_at_20_max
|
|
value: 23.024130858142723
|
|
- type: nauc_precision_at_20_std
|
|
value: 13.739145648899603
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 30.11583572134629
|
|
- type: nauc_precision_at_3_max
|
|
value: 31.37321080069495
|
|
- type: nauc_precision_at_3_std
|
|
value: 4.705512374126024
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 23.192015335996093
|
|
- type: nauc_precision_at_5_max
|
|
value: 29.415746835998764
|
|
- type: nauc_precision_at_5_std
|
|
value: 6.843498772798558
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 25.36573313426033
|
|
- type: nauc_recall_at_1000_max
|
|
value: 43.06672256524168
|
|
- type: nauc_recall_at_1000_std
|
|
value: 47.93664853815292
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 31.222880916617406
|
|
- type: nauc_recall_at_100_max
|
|
value: 31.761159904172658
|
|
- type: nauc_recall_at_100_std
|
|
value: 23.034218976635877
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 36.23439028915225
|
|
- type: nauc_recall_at_10_max
|
|
value: 28.473458977606438
|
|
- type: nauc_recall_at_10_std
|
|
value: 3.7797969934159
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 52.48913346466124
|
|
- type: nauc_recall_at_1_max
|
|
value: 29.874374024967725
|
|
- type: nauc_recall_at_1_std
|
|
value: -2.433547569836134
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 34.678676952584766
|
|
- type: nauc_recall_at_20_max
|
|
value: 29.04638392522168
|
|
- type: nauc_recall_at_20_std
|
|
value: 8.148894982082549
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 41.31029996231311
|
|
- type: nauc_recall_at_3_max
|
|
value: 28.44199443414157
|
|
- type: nauc_recall_at_3_std
|
|
value: -0.747324057600377
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 38.535873899920674
|
|
- type: nauc_recall_at_5_max
|
|
value: 27.942667805948375
|
|
- type: nauc_recall_at_5_std
|
|
value: 0.30652206930973686
|
|
- type: ndcg_at_1
|
|
value: 36.47675
|
|
- type: ndcg_at_10
|
|
value: 47.74883333333334
|
|
- type: ndcg_at_100
|
|
value: 52.902416666666674
|
|
- type: ndcg_at_1000
|
|
value: 54.69116666666667
|
|
- type: ndcg_at_20
|
|
value: 49.89758333333333
|
|
- type: ndcg_at_3
|
|
value: 42.462250000000004
|
|
- type: ndcg_at_5
|
|
value: 44.91841666666667
|
|
- type: precision_at_1
|
|
value: 36.47675
|
|
- type: precision_at_10
|
|
value: 8.582416666666665
|
|
- type: precision_at_100
|
|
value: 1.31475
|
|
- type: precision_at_1000
|
|
value: 0.16458333333333333
|
|
- type: precision_at_20
|
|
value: 5.021833333333333
|
|
- type: precision_at_3
|
|
value: 20.004499999999997
|
|
- type: precision_at_5
|
|
value: 14.178666666666665
|
|
- type: recall_at_1
|
|
value: 30.179249999999996
|
|
- type: recall_at_10
|
|
value: 60.950166666666675
|
|
- type: recall_at_100
|
|
value: 83.19025
|
|
- type: recall_at_1000
|
|
value: 95.27774999999998
|
|
- type: recall_at_20
|
|
value: 68.80175
|
|
- type: recall_at_3
|
|
value: 46.01841666666666
|
|
- type: recall_at_5
|
|
value: 52.482416666666666
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB ClimateFEVER
|
|
revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380
|
|
split: test
|
|
type: mteb/climate-fever
|
|
metrics:
|
|
- type: main_score
|
|
value: 46.113
|
|
- type: map_at_1
|
|
value: 20.122999999999998
|
|
- type: map_at_10
|
|
value: 35.474
|
|
- type: map_at_100
|
|
value: 37.592
|
|
- type: map_at_1000
|
|
value: 37.773
|
|
- type: map_at_20
|
|
value: 36.637
|
|
- type: map_at_3
|
|
value: 29.731
|
|
- type: map_at_5
|
|
value: 32.964
|
|
- type: mrr_at_1
|
|
value: 46.71009771986971
|
|
- type: mrr_at_10
|
|
value: 58.855669303552105
|
|
- type: mrr_at_100
|
|
value: 59.389249674038425
|
|
- type: mrr_at_1000
|
|
value: 59.408448104362364
|
|
- type: mrr_at_20
|
|
value: 59.23881203149016
|
|
- type: mrr_at_3
|
|
value: 56.18892508143328
|
|
- type: mrr_at_5
|
|
value: 57.85342019543985
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 27.047031037721958
|
|
- type: nauc_map_at_1000_max
|
|
value: 43.25240279148033
|
|
- type: nauc_map_at_1000_std
|
|
value: 20.795849418696037
|
|
- type: nauc_map_at_100_diff1
|
|
value: 27.044739015116452
|
|
- type: nauc_map_at_100_max
|
|
value: 43.24042159787812
|
|
- type: nauc_map_at_100_std
|
|
value: 20.799952124137683
|
|
- type: nauc_map_at_10_diff1
|
|
value: 27.372696854670338
|
|
- type: nauc_map_at_10_max
|
|
value: 43.054456574721684
|
|
- type: nauc_map_at_10_std
|
|
value: 19.537162110136645
|
|
- type: nauc_map_at_1_diff1
|
|
value: 43.65424623953092
|
|
- type: nauc_map_at_1_max
|
|
value: 45.17986509998762
|
|
- type: nauc_map_at_1_std
|
|
value: 8.497107052335414
|
|
- type: nauc_map_at_20_diff1
|
|
value: 27.224535846566074
|
|
- type: nauc_map_at_20_max
|
|
value: 43.12222854561229
|
|
- type: nauc_map_at_20_std
|
|
value: 20.29982972202669
|
|
- type: nauc_map_at_3_diff1
|
|
value: 30.87847002319001
|
|
- type: nauc_map_at_3_max
|
|
value: 42.890027891707575
|
|
- type: nauc_map_at_3_std
|
|
value: 13.857451947580929
|
|
- type: nauc_map_at_5_diff1
|
|
value: 27.966867093591542
|
|
- type: nauc_map_at_5_max
|
|
value: 42.35826637592201
|
|
- type: nauc_map_at_5_std
|
|
value: 16.993102524058624
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 30.191544077608164
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 44.959438920351644
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 24.065801376465114
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 30.170368115494
|
|
- type: nauc_mrr_at_100_max
|
|
value: 44.955868115761156
|
|
- type: nauc_mrr_at_100_std
|
|
value: 24.093510767847707
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 30.128430637520175
|
|
- type: nauc_mrr_at_10_max
|
|
value: 44.97689261350708
|
|
- type: nauc_mrr_at_10_std
|
|
value: 24.037049561818897
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 35.323351939108214
|
|
- type: nauc_mrr_at_1_max
|
|
value: 43.85026244855636
|
|
- type: nauc_mrr_at_1_std
|
|
value: 17.040662141218974
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 30.192006556160443
|
|
- type: nauc_mrr_at_20_max
|
|
value: 45.02814530774032
|
|
- type: nauc_mrr_at_20_std
|
|
value: 24.20885865448696
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 29.88250163424518
|
|
- type: nauc_mrr_at_3_max
|
|
value: 44.25768944883186
|
|
- type: nauc_mrr_at_3_std
|
|
value: 22.804183393364198
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 30.269824490420767
|
|
- type: nauc_mrr_at_5_max
|
|
value: 44.97443265796657
|
|
- type: nauc_mrr_at_5_std
|
|
value: 23.894159916141177
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 24.533764005407356
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 44.50902713386608
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 27.589506980238404
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 24.209785073940353
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 44.18257063893669
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 27.963150866401943
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 25.168069201989486
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 43.84940910683214
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 24.810707270956435
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 35.323351939108214
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 43.85026244855636
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 17.040662141218974
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 24.829924800466834
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 43.738574327059716
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 26.252370278684072
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 27.321943393906274
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 42.16584786993447
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 18.24775079455969
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 26.043785418347998
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 42.874593895388344
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 21.294004555506117
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -22.073027615308582
|
|
- type: nauc_precision_at_1000_max
|
|
value: -6.549723766317357
|
|
- type: nauc_precision_at_1000_std
|
|
value: 18.301749191241306
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -15.654286887593619
|
|
- type: nauc_precision_at_100_max
|
|
value: 6.401516251421999
|
|
- type: nauc_precision_at_100_std
|
|
value: 29.170680324929805
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -4.362381972892247
|
|
- type: nauc_precision_at_10_max
|
|
value: 22.10943515872447
|
|
- type: nauc_precision_at_10_std
|
|
value: 31.869699459530022
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 35.323351939108214
|
|
- type: nauc_precision_at_1_max
|
|
value: 43.85026244855636
|
|
- type: nauc_precision_at_1_std
|
|
value: 17.040662141218974
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -7.50749661117875
|
|
- type: nauc_precision_at_20_max
|
|
value: 16.80584016023257
|
|
- type: nauc_precision_at_20_std
|
|
value: 31.976755897112437
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 7.402667538773083
|
|
- type: nauc_precision_at_3_max
|
|
value: 31.2088401330676
|
|
- type: nauc_precision_at_3_std
|
|
value: 24.287905698405662
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 0.7479172565343901
|
|
- type: nauc_precision_at_5_max
|
|
value: 26.28427734237825
|
|
- type: nauc_precision_at_5_std
|
|
value: 28.246947120310317
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 2.4778431086370496
|
|
- type: nauc_recall_at_1000_max
|
|
value: 40.2231995797509
|
|
- type: nauc_recall_at_1000_std
|
|
value: 52.62124052183862
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 8.960962419741463
|
|
- type: nauc_recall_at_100_max
|
|
value: 35.81132850291491
|
|
- type: nauc_recall_at_100_std
|
|
value: 40.020903251786166
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 15.603400751376636
|
|
- type: nauc_recall_at_10_max
|
|
value: 37.570127529136485
|
|
- type: nauc_recall_at_10_std
|
|
value: 28.07128410238545
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 43.65424623953092
|
|
- type: nauc_recall_at_1_max
|
|
value: 45.17986509998762
|
|
- type: nauc_recall_at_1_std
|
|
value: 8.497107052335414
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 13.844820282832346
|
|
- type: nauc_recall_at_20_max
|
|
value: 36.0106148516309
|
|
- type: nauc_recall_at_20_std
|
|
value: 31.453103910565254
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 24.359328154117748
|
|
- type: nauc_recall_at_3_max
|
|
value: 39.93774251377568
|
|
- type: nauc_recall_at_3_std
|
|
value: 16.214921517509648
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 18.75788451360292
|
|
- type: nauc_recall_at_5_max
|
|
value: 38.177646107055516
|
|
- type: nauc_recall_at_5_std
|
|
value: 22.17196825834675
|
|
- type: ndcg_at_1
|
|
value: 46.71
|
|
- type: ndcg_at_10
|
|
value: 46.113
|
|
- type: ndcg_at_100
|
|
value: 53.035
|
|
- type: ndcg_at_1000
|
|
value: 55.724
|
|
- type: ndcg_at_20
|
|
value: 48.929
|
|
- type: ndcg_at_3
|
|
value: 39.501999999999995
|
|
- type: ndcg_at_5
|
|
value: 41.792
|
|
- type: precision_at_1
|
|
value: 46.71
|
|
- type: precision_at_10
|
|
value: 14.274000000000001
|
|
- type: precision_at_100
|
|
value: 2.1870000000000003
|
|
- type: precision_at_1000
|
|
value: 0.269
|
|
- type: precision_at_20
|
|
value: 8.375
|
|
- type: precision_at_3
|
|
value: 29.881
|
|
- type: precision_at_5
|
|
value: 22.697
|
|
- type: recall_at_1
|
|
value: 20.122999999999998
|
|
- type: recall_at_10
|
|
value: 52.22
|
|
- type: recall_at_100
|
|
value: 75.388
|
|
- type: recall_at_1000
|
|
value: 89.938
|
|
- type: recall_at_20
|
|
value: 60.077000000000005
|
|
- type: recall_at_3
|
|
value: 35.150999999999996
|
|
- type: recall_at_5
|
|
value: 42.748000000000005
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB DBPedia
|
|
revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659
|
|
split: test
|
|
type: mteb/dbpedia
|
|
metrics:
|
|
- type: main_score
|
|
value: 52.276999999999994
|
|
- type: map_at_1
|
|
value: 9.949
|
|
- type: map_at_10
|
|
value: 24.891
|
|
- type: map_at_100
|
|
value: 37.111
|
|
- type: map_at_1000
|
|
value: 39.266
|
|
- type: map_at_20
|
|
value: 29.685
|
|
- type: map_at_3
|
|
value: 16.586000000000002
|
|
- type: map_at_5
|
|
value: 19.982
|
|
- type: mrr_at_1
|
|
value: 76.25
|
|
- type: mrr_at_10
|
|
value: 82.4518849206349
|
|
- type: mrr_at_100
|
|
value: 82.70302194564499
|
|
- type: mrr_at_1000
|
|
value: 82.70909729942254
|
|
- type: mrr_at_20
|
|
value: 82.60492765962964
|
|
- type: mrr_at_3
|
|
value: 81.33333333333331
|
|
- type: mrr_at_5
|
|
value: 82.14583333333331
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 21.427201262456556
|
|
- type: nauc_map_at_1000_max
|
|
value: 35.357361590816076
|
|
- type: nauc_map_at_1000_std
|
|
value: 24.785419223353717
|
|
- type: nauc_map_at_100_diff1
|
|
value: 22.82358692021537
|
|
- type: nauc_map_at_100_max
|
|
value: 35.07399692072945
|
|
- type: nauc_map_at_100_std
|
|
value: 22.679878828987025
|
|
- type: nauc_map_at_10_diff1
|
|
value: 26.491769223479643
|
|
- type: nauc_map_at_10_max
|
|
value: 20.78079385443902
|
|
- type: nauc_map_at_10_std
|
|
value: -4.910406292079661
|
|
- type: nauc_map_at_1_diff1
|
|
value: 35.20851030208876
|
|
- type: nauc_map_at_1_max
|
|
value: 5.783003346365858
|
|
- type: nauc_map_at_1_std
|
|
value: -21.11679133835354
|
|
- type: nauc_map_at_20_diff1
|
|
value: 24.80097499300491
|
|
- type: nauc_map_at_20_max
|
|
value: 26.807021360774975
|
|
- type: nauc_map_at_20_std
|
|
value: 4.793103995429955
|
|
- type: nauc_map_at_3_diff1
|
|
value: 29.238193458890173
|
|
- type: nauc_map_at_3_max
|
|
value: 10.300839972189456
|
|
- type: nauc_map_at_3_std
|
|
value: -17.889666731981592
|
|
- type: nauc_map_at_5_diff1
|
|
value: 28.773624870573926
|
|
- type: nauc_map_at_5_max
|
|
value: 14.951435645422887
|
|
- type: nauc_map_at_5_std
|
|
value: -13.319697827173565
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 55.232544856708785
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 64.73225637682637
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 37.57480399594188
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 55.219251601773735
|
|
- type: nauc_mrr_at_100_max
|
|
value: 64.73305063663611
|
|
- type: nauc_mrr_at_100_std
|
|
value: 37.56458562909293
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 55.123463838253464
|
|
- type: nauc_mrr_at_10_max
|
|
value: 64.91914041040233
|
|
- type: nauc_mrr_at_10_std
|
|
value: 37.76482503851598
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 56.45461238513347
|
|
- type: nauc_mrr_at_1_max
|
|
value: 63.11782510293676
|
|
- type: nauc_mrr_at_1_std
|
|
value: 33.592561284868985
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 55.15401961460458
|
|
- type: nauc_mrr_at_20_max
|
|
value: 64.77145835613156
|
|
- type: nauc_mrr_at_20_std
|
|
value: 37.471561418305804
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 54.64387438697658
|
|
- type: nauc_mrr_at_3_max
|
|
value: 64.27618995019164
|
|
- type: nauc_mrr_at_3_std
|
|
value: 39.391637295269014
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 55.08702591239485
|
|
- type: nauc_mrr_at_5_max
|
|
value: 64.6071475650635
|
|
- type: nauc_mrr_at_5_std
|
|
value: 37.97185134269896
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 31.696698876400387
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 52.12183760001191
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 40.197596211778716
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 33.253120193433666
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 49.47167758554746
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 32.643833139756204
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 27.065541392580013
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 45.83504281289289
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 27.11739500732328
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 49.42808250022517
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 53.502615048520354
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 27.17555908836708
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 29.374791382330308
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 43.91246842479055
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 23.419410620550316
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 26.71550354496204
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 43.9641457892003
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 27.320024167947686
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 27.020654974589487
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 46.130417266030584
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 28.392009019010068
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -21.47455482181002
|
|
- type: nauc_precision_at_1000_max
|
|
value: -9.721907229236024
|
|
- type: nauc_precision_at_1000_std
|
|
value: -1.061132062651487
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -12.35759246101943
|
|
- type: nauc_precision_at_100_max
|
|
value: 15.509512444892168
|
|
- type: nauc_precision_at_100_std
|
|
value: 36.21183578592014
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -6.136998947343125
|
|
- type: nauc_precision_at_10_max
|
|
value: 32.30037906748288
|
|
- type: nauc_precision_at_10_std
|
|
value: 41.4500302476981
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 56.45461238513347
|
|
- type: nauc_precision_at_1_max
|
|
value: 63.11782510293676
|
|
- type: nauc_precision_at_1_std
|
|
value: 33.592561284868985
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -7.335890123683174
|
|
- type: nauc_precision_at_20_max
|
|
value: 28.31417075291312
|
|
- type: nauc_precision_at_20_std
|
|
value: 41.405935715061815
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 7.117255890225942
|
|
- type: nauc_precision_at_3_max
|
|
value: 39.19894132683829
|
|
- type: nauc_precision_at_3_std
|
|
value: 38.48255841994843
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 1.861523090114206
|
|
- type: nauc_precision_at_5_max
|
|
value: 38.11649223007208
|
|
- type: nauc_precision_at_5_std
|
|
value: 40.52993530374645
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 26.497648584314636
|
|
- type: nauc_recall_at_1000_max
|
|
value: 44.48069746734414
|
|
- type: nauc_recall_at_1000_std
|
|
value: 53.16438130228715
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 26.353456899511446
|
|
- type: nauc_recall_at_100_max
|
|
value: 37.57379787884197
|
|
- type: nauc_recall_at_100_std
|
|
value: 29.197468295989548
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 22.80445738351114
|
|
- type: nauc_recall_at_10_max
|
|
value: 15.895630778449046
|
|
- type: nauc_recall_at_10_std
|
|
value: -8.746224797644501
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 35.20851030208876
|
|
- type: nauc_recall_at_1_max
|
|
value: 5.783003346365858
|
|
- type: nauc_recall_at_1_std
|
|
value: -21.11679133835354
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 22.34028867678706
|
|
- type: nauc_recall_at_20_max
|
|
value: 21.42373427646772
|
|
- type: nauc_recall_at_20_std
|
|
value: 0.4533036151015875
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 24.96853445599229
|
|
- type: nauc_recall_at_3_max
|
|
value: 6.245185375804208
|
|
- type: nauc_recall_at_3_std
|
|
value: -20.200240127099622
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 24.749259476710623
|
|
- type: nauc_recall_at_5_max
|
|
value: 11.024592845995942
|
|
- type: nauc_recall_at_5_std
|
|
value: -16.15683085641543
|
|
- type: ndcg_at_1
|
|
value: 64.125
|
|
- type: ndcg_at_10
|
|
value: 52.276999999999994
|
|
- type: ndcg_at_100
|
|
value: 57.440000000000005
|
|
- type: ndcg_at_1000
|
|
value: 64.082
|
|
- type: ndcg_at_20
|
|
value: 51.383
|
|
- type: ndcg_at_3
|
|
value: 55.769000000000005
|
|
- type: ndcg_at_5
|
|
value: 53.978
|
|
- type: precision_at_1
|
|
value: 76.25
|
|
- type: precision_at_10
|
|
value: 43.05
|
|
- type: precision_at_100
|
|
value: 14.09
|
|
- type: precision_at_1000
|
|
value: 2.662
|
|
- type: precision_at_20
|
|
value: 33.112
|
|
- type: precision_at_3
|
|
value: 59.833000000000006
|
|
- type: precision_at_5
|
|
value: 53.05
|
|
- type: recall_at_1
|
|
value: 9.949
|
|
- type: recall_at_10
|
|
value: 30.424
|
|
- type: recall_at_100
|
|
value: 64.062
|
|
- type: recall_at_1000
|
|
value: 85.916
|
|
- type: recall_at_20
|
|
value: 39.895
|
|
- type: recall_at_3
|
|
value: 17.876
|
|
- type: recall_at_5
|
|
value: 22.536
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB EmotionClassification
|
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37
|
|
split: test
|
|
type: mteb/emotion
|
|
metrics:
|
|
- type: accuracy
|
|
value: 84.29499999999999
|
|
- type: f1
|
|
value: 79.76188258172078
|
|
- type: f1_weighted
|
|
value: 84.96026012933847
|
|
- type: main_score
|
|
value: 84.29499999999999
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB FEVER
|
|
revision: bea83ef9e8fb933d90a2f1d5515737465d613e12
|
|
split: test
|
|
type: mteb/fever
|
|
metrics:
|
|
- type: main_score
|
|
value: 94.83200000000001
|
|
- type: map_at_1
|
|
value: 87.339
|
|
- type: map_at_10
|
|
value: 92.92099999999999
|
|
- type: map_at_100
|
|
value: 93.108
|
|
- type: map_at_1000
|
|
value: 93.116
|
|
- type: map_at_20
|
|
value: 93.041
|
|
- type: map_at_3
|
|
value: 92.219
|
|
- type: map_at_5
|
|
value: 92.664
|
|
- type: mrr_at_1
|
|
value: 93.99939993999399
|
|
- type: mrr_at_10
|
|
value: 96.55188137861403
|
|
- type: mrr_at_100
|
|
value: 96.5652366009286
|
|
- type: mrr_at_1000
|
|
value: 96.5652625550811
|
|
- type: mrr_at_20
|
|
value: 96.5601781754844
|
|
- type: mrr_at_3
|
|
value: 96.45714571457142
|
|
- type: mrr_at_5
|
|
value: 96.544904490449
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 51.81676454961933
|
|
- type: nauc_map_at_1000_max
|
|
value: 24.904822914926118
|
|
- type: nauc_map_at_1000_std
|
|
value: -3.8110347821630404
|
|
- type: nauc_map_at_100_diff1
|
|
value: 51.77514975011158
|
|
- type: nauc_map_at_100_max
|
|
value: 24.912497341800094
|
|
- type: nauc_map_at_100_std
|
|
value: -3.76229517662447
|
|
- type: nauc_map_at_10_diff1
|
|
value: 51.29608296382479
|
|
- type: nauc_map_at_10_max
|
|
value: 24.78704970246707
|
|
- type: nauc_map_at_10_std
|
|
value: -3.723130815783328
|
|
- type: nauc_map_at_1_diff1
|
|
value: 59.90813138005125
|
|
- type: nauc_map_at_1_max
|
|
value: 24.58479295693794
|
|
- type: nauc_map_at_1_std
|
|
value: -8.056152492777027
|
|
- type: nauc_map_at_20_diff1
|
|
value: 51.428639331678326
|
|
- type: nauc_map_at_20_max
|
|
value: 24.849214517705086
|
|
- type: nauc_map_at_20_std
|
|
value: -3.685550123874596
|
|
- type: nauc_map_at_3_diff1
|
|
value: 50.94399923719279
|
|
- type: nauc_map_at_3_max
|
|
value: 24.359700180006207
|
|
- type: nauc_map_at_3_std
|
|
value: -5.407767408816422
|
|
- type: nauc_map_at_5_diff1
|
|
value: 50.767302682959546
|
|
- type: nauc_map_at_5_max
|
|
value: 24.491113461892215
|
|
- type: nauc_map_at_5_std
|
|
value: -4.058336127339082
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 79.86042313551833
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 23.20960445633933
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -23.54334295120471
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 79.85991247027636
|
|
- type: nauc_mrr_at_100_max
|
|
value: 23.210085926780106
|
|
- type: nauc_mrr_at_100_std
|
|
value: -23.542508200789197
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 79.71095155563415
|
|
- type: nauc_mrr_at_10_max
|
|
value: 23.24128650883908
|
|
- type: nauc_mrr_at_10_std
|
|
value: -23.408502781834102
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 82.6349900233902
|
|
- type: nauc_mrr_at_1_max
|
|
value: 21.994548214014227
|
|
- type: nauc_mrr_at_1_std
|
|
value: -22.549769792179262
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 79.76465012873038
|
|
- type: nauc_mrr_at_20_max
|
|
value: 23.17575026523213
|
|
- type: nauc_mrr_at_20_std
|
|
value: -23.492660166315048
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 79.91074933379953
|
|
- type: nauc_mrr_at_3_max
|
|
value: 24.14246499097892
|
|
- type: nauc_mrr_at_3_std
|
|
value: -25.22601708389664
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 79.62092651565847
|
|
- type: nauc_mrr_at_5_max
|
|
value: 23.315937737034425
|
|
- type: nauc_mrr_at_5_std
|
|
value: -23.317659360058403
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 54.404537986779225
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 25.38408304128995
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -4.916709117696968
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 53.2448598868241
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 25.75325255295546
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -3.680507005630751
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 50.81057355170232
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 25.006448273343807
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -2.8979899112515577
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 82.6349900233902
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 21.994548214014227
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -22.549769792179262
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 51.205023097166304
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 25.22133626556826
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -2.9506328244150155
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 51.79780256736321
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 24.81137324438439
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -6.881223858227807
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 50.290038260564565
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 24.57250792165796
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -3.5124628344654596
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -20.215211396894333
|
|
- type: nauc_precision_at_1000_max
|
|
value: -14.165452298769171
|
|
- type: nauc_precision_at_1000_std
|
|
value: -2.0952871214470816
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -22.340257474494607
|
|
- type: nauc_precision_at_100_max
|
|
value: -12.697885641360282
|
|
- type: nauc_precision_at_100_std
|
|
value: 1.0688624940286244
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -24.78271817420798
|
|
- type: nauc_precision_at_10_max
|
|
value: -12.625257500222656
|
|
- type: nauc_precision_at_10_std
|
|
value: 3.223250450607087
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 82.6349900233902
|
|
- type: nauc_precision_at_1_max
|
|
value: 21.994548214014227
|
|
- type: nauc_precision_at_1_std
|
|
value: -22.549769792179262
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -24.375756227194177
|
|
- type: nauc_precision_at_20_max
|
|
value: -12.341015011563536
|
|
- type: nauc_precision_at_20_std
|
|
value: 2.7475274619387955
|
|
- type: nauc_precision_at_3_diff1
|
|
value: -24.8251306777365
|
|
- type: nauc_precision_at_3_max
|
|
value: -13.109579709589042
|
|
- type: nauc_precision_at_3_std
|
|
value: -1.2233442335420748
|
|
- type: nauc_precision_at_5_diff1
|
|
value: -26.955418583344894
|
|
- type: nauc_precision_at_5_max
|
|
value: -13.598630838071015
|
|
- type: nauc_precision_at_5_std
|
|
value: 2.545780631940738
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 0.2542680835344437
|
|
- type: nauc_recall_at_1000_max
|
|
value: 49.38194243035277
|
|
- type: nauc_recall_at_1000_std
|
|
value: 57.021502715846026
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 5.062154815367015
|
|
- type: nauc_recall_at_100_max
|
|
value: 45.41178380188437
|
|
- type: nauc_recall_at_100_std
|
|
value: 50.78382225901813
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 20.429153629007818
|
|
- type: nauc_recall_at_10_max
|
|
value: 27.516855026155508
|
|
- type: nauc_recall_at_10_std
|
|
value: 21.367491371755467
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 59.90813138005125
|
|
- type: nauc_recall_at_1_max
|
|
value: 24.58479295693794
|
|
- type: nauc_recall_at_1_std
|
|
value: -8.056152492777027
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 13.072430858896942
|
|
- type: nauc_recall_at_20_max
|
|
value: 29.5522659183247
|
|
- type: nauc_recall_at_20_std
|
|
value: 28.70569974090291
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 30.419084482663617
|
|
- type: nauc_recall_at_3_max
|
|
value: 25.627389580252835
|
|
- type: nauc_recall_at_3_std
|
|
value: 2.5557690877637054
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 22.92561435069869
|
|
- type: nauc_recall_at_5_max
|
|
value: 25.545265063475455
|
|
- type: nauc_recall_at_5_std
|
|
value: 14.736172663072786
|
|
- type: ndcg_at_1
|
|
value: 93.999
|
|
- type: ndcg_at_10
|
|
value: 94.83200000000001
|
|
- type: ndcg_at_100
|
|
value: 95.363
|
|
- type: ndcg_at_1000
|
|
value: 95.478
|
|
- type: ndcg_at_20
|
|
value: 95.077
|
|
- type: ndcg_at_3
|
|
value: 94.143
|
|
- type: ndcg_at_5
|
|
value: 94.525
|
|
- type: precision_at_1
|
|
value: 93.999
|
|
- type: precision_at_10
|
|
value: 11.029
|
|
- type: precision_at_100
|
|
value: 1.1560000000000001
|
|
- type: precision_at_1000
|
|
value: 0.11800000000000001
|
|
- type: precision_at_20
|
|
value: 5.62
|
|
- type: precision_at_3
|
|
value: 35.219
|
|
- type: precision_at_5
|
|
value: 21.584
|
|
- type: recall_at_1
|
|
value: 87.339
|
|
- type: recall_at_10
|
|
value: 97.026
|
|
- type: recall_at_100
|
|
value: 98.936
|
|
- type: recall_at_1000
|
|
value: 99.599
|
|
- type: recall_at_20
|
|
value: 97.744
|
|
- type: recall_at_3
|
|
value: 95.069
|
|
- type: recall_at_5
|
|
value: 96.177
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB FiQA2018
|
|
revision: 27a168819829fe9bcd655c2df245fb19452e8e06
|
|
split: test
|
|
type: mteb/fiqa
|
|
metrics:
|
|
- type: main_score
|
|
value: 60.480000000000004
|
|
- type: map_at_1
|
|
value: 31.529
|
|
- type: map_at_10
|
|
value: 52.081
|
|
- type: map_at_100
|
|
value: 54.342
|
|
- type: map_at_1000
|
|
value: 54.449000000000005
|
|
- type: map_at_20
|
|
value: 53.479
|
|
- type: map_at_3
|
|
value: 45.471000000000004
|
|
- type: map_at_5
|
|
value: 49.164
|
|
- type: mrr_at_1
|
|
value: 60.03086419753087
|
|
- type: mrr_at_10
|
|
value: 67.73754409171075
|
|
- type: mrr_at_100
|
|
value: 68.332432152368
|
|
- type: mrr_at_1000
|
|
value: 68.34150941774908
|
|
- type: mrr_at_20
|
|
value: 68.14780993838725
|
|
- type: mrr_at_3
|
|
value: 65.6378600823045
|
|
- type: mrr_at_5
|
|
value: 66.88014403292176
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 45.36598134579052
|
|
- type: nauc_map_at_1000_max
|
|
value: 31.891451119906943
|
|
- type: nauc_map_at_1000_std
|
|
value: -15.41454384137943
|
|
- type: nauc_map_at_100_diff1
|
|
value: 45.31268291874018
|
|
- type: nauc_map_at_100_max
|
|
value: 31.811055683002092
|
|
- type: nauc_map_at_100_std
|
|
value: -15.348503855591417
|
|
- type: nauc_map_at_10_diff1
|
|
value: 45.22606983565892
|
|
- type: nauc_map_at_10_max
|
|
value: 30.46108534749699
|
|
- type: nauc_map_at_10_std
|
|
value: -16.618086029682555
|
|
- type: nauc_map_at_1_diff1
|
|
value: 49.94952823753276
|
|
- type: nauc_map_at_1_max
|
|
value: 13.770377574254548
|
|
- type: nauc_map_at_1_std
|
|
value: -14.946357968858653
|
|
- type: nauc_map_at_20_diff1
|
|
value: 45.29274207897926
|
|
- type: nauc_map_at_20_max
|
|
value: 31.27332015148257
|
|
- type: nauc_map_at_20_std
|
|
value: -15.782946115613129
|
|
- type: nauc_map_at_3_diff1
|
|
value: 47.94248233566038
|
|
- type: nauc_map_at_3_max
|
|
value: 24.022838776825456
|
|
- type: nauc_map_at_3_std
|
|
value: -17.103518542262208
|
|
- type: nauc_map_at_5_diff1
|
|
value: 45.85345590031722
|
|
- type: nauc_map_at_5_max
|
|
value: 27.78341379004547
|
|
- type: nauc_map_at_5_std
|
|
value: -17.490850791756326
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 58.225141047822824
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 43.39606904140525
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -14.64093518199122
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 58.22137274179545
|
|
- type: nauc_mrr_at_100_max
|
|
value: 43.39567568136935
|
|
- type: nauc_mrr_at_100_std
|
|
value: -14.62512313985582
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 58.03217329957151
|
|
- type: nauc_mrr_at_10_max
|
|
value: 43.633561683075186
|
|
- type: nauc_mrr_at_10_std
|
|
value: -14.563703576023808
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 61.48979902647692
|
|
- type: nauc_mrr_at_1_max
|
|
value: 43.1938079066948
|
|
- type: nauc_mrr_at_1_std
|
|
value: -15.808138277440465
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 58.13185370150794
|
|
- type: nauc_mrr_at_20_max
|
|
value: 43.35607721183147
|
|
- type: nauc_mrr_at_20_std
|
|
value: -14.635812702971263
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 58.698963168321264
|
|
- type: nauc_mrr_at_3_max
|
|
value: 43.633129249785405
|
|
- type: nauc_mrr_at_3_std
|
|
value: -15.733246346983854
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 57.94156745229547
|
|
- type: nauc_mrr_at_5_max
|
|
value: 43.14152462640525
|
|
- type: nauc_mrr_at_5_std
|
|
value: -15.318685307750895
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 47.871896043731496
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 37.159845167533426
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -13.067288160833485
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 47.046171407204426
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 36.422514360855835
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -11.636859259571441
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 46.232628149078096
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 34.82402625088358
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -14.768545542980114
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 61.48979902647692
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 43.1938079066948
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -15.808138277440465
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 46.51116172390955
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 35.36362650568298
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -12.849406209182826
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 47.39832263785871
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 35.67466264628456
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -17.257717349296943
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 45.91049493804232
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 33.8405091138445
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -17.477069902735895
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -12.037873000917767
|
|
- type: nauc_precision_at_1000_max
|
|
value: 26.043220150002295
|
|
- type: nauc_precision_at_1000_std
|
|
value: 6.84910668321572
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -9.383403459051864
|
|
- type: nauc_precision_at_100_max
|
|
value: 29.68713170610003
|
|
- type: nauc_precision_at_100_std
|
|
value: 10.079531587056152
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 3.3433323353925135
|
|
- type: nauc_precision_at_10_max
|
|
value: 38.31790111725993
|
|
- type: nauc_precision_at_10_std
|
|
value: 0.7888123304710856
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 61.48979902647692
|
|
- type: nauc_precision_at_1_max
|
|
value: 43.1938079066948
|
|
- type: nauc_precision_at_1_std
|
|
value: -15.808138277440465
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -2.083500986294448
|
|
- type: nauc_precision_at_20_max
|
|
value: 35.77143835726343
|
|
- type: nauc_precision_at_20_std
|
|
value: 5.318547021874003
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 23.335617788912586
|
|
- type: nauc_precision_at_3_max
|
|
value: 39.81973275320871
|
|
- type: nauc_precision_at_3_std
|
|
value: -8.442769390555561
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 11.521087842589482
|
|
- type: nauc_precision_at_5_max
|
|
value: 39.527792539828255
|
|
- type: nauc_precision_at_5_std
|
|
value: -5.412729503701626
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 10.6830893047453
|
|
- type: nauc_recall_at_1000_max
|
|
value: 8.834504311238423
|
|
- type: nauc_recall_at_1000_std
|
|
value: 24.670754304859692
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 20.646020385527358
|
|
- type: nauc_recall_at_100_max
|
|
value: 20.121595011523294
|
|
- type: nauc_recall_at_100_std
|
|
value: 19.42307459311791
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 33.01029313733417
|
|
- type: nauc_recall_at_10_max
|
|
value: 27.948634980368702
|
|
- type: nauc_recall_at_10_std
|
|
value: -10.239767371462975
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 49.94952823753276
|
|
- type: nauc_recall_at_1_max
|
|
value: 13.770377574254548
|
|
- type: nauc_recall_at_1_std
|
|
value: -14.946357968858653
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 30.040111045267963
|
|
- type: nauc_recall_at_20_max
|
|
value: 25.984919302418184
|
|
- type: nauc_recall_at_20_std
|
|
value: -1.4998001817460804
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 42.24410559113653
|
|
- type: nauc_recall_at_3_max
|
|
value: 20.269503583626914
|
|
- type: nauc_recall_at_3_std
|
|
value: -17.09578532600584
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 36.124149735848945
|
|
- type: nauc_recall_at_5_max
|
|
value: 22.708022306002622
|
|
- type: nauc_recall_at_5_std
|
|
value: -16.966976847236193
|
|
- type: ndcg_at_1
|
|
value: 60.031
|
|
- type: ndcg_at_10
|
|
value: 60.480000000000004
|
|
- type: ndcg_at_100
|
|
value: 66.94099999999999
|
|
- type: ndcg_at_1000
|
|
value: 68.303
|
|
- type: ndcg_at_20
|
|
value: 63.536
|
|
- type: ndcg_at_3
|
|
value: 55.903999999999996
|
|
- type: ndcg_at_5
|
|
value: 57.387
|
|
- type: precision_at_1
|
|
value: 60.031
|
|
- type: precision_at_10
|
|
value: 16.682
|
|
- type: precision_at_100
|
|
value: 2.336
|
|
- type: precision_at_1000
|
|
value: 0.259
|
|
- type: precision_at_20
|
|
value: 9.66
|
|
- type: precision_at_3
|
|
value: 37.191
|
|
- type: precision_at_5
|
|
value: 27.253
|
|
- type: recall_at_1
|
|
value: 31.529
|
|
- type: recall_at_10
|
|
value: 68.035
|
|
- type: recall_at_100
|
|
value: 90.925
|
|
- type: recall_at_1000
|
|
value: 98.688
|
|
- type: recall_at_20
|
|
value: 77.453
|
|
- type: recall_at_3
|
|
value: 50.221000000000004
|
|
- type: recall_at_5
|
|
value: 58.209999999999994
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB HotpotQA
|
|
revision: ab518f4d6fcca38d87c25209f94beba119d02014
|
|
split: test
|
|
type: mteb/hotpotqa
|
|
metrics:
|
|
- type: main_score
|
|
value: 76.67399999999999
|
|
- type: map_at_1
|
|
value: 43.822
|
|
- type: map_at_10
|
|
value: 68.82000000000001
|
|
- type: map_at_100
|
|
value: 69.659
|
|
- type: map_at_1000
|
|
value: 69.714
|
|
- type: map_at_20
|
|
value: 69.305
|
|
- type: map_at_3
|
|
value: 65.517
|
|
- type: map_at_5
|
|
value: 67.633
|
|
- type: mrr_at_1
|
|
value: 87.643484132343
|
|
- type: mrr_at_10
|
|
value: 91.28134679485098
|
|
- type: mrr_at_100
|
|
value: 91.37985230614755
|
|
- type: mrr_at_1000
|
|
value: 91.38202467630681
|
|
- type: mrr_at_20
|
|
value: 91.34718855278429
|
|
- type: mrr_at_3
|
|
value: 90.75849651136599
|
|
- type: mrr_at_5
|
|
value: 91.10961062345235
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 3.7670405082837477
|
|
- type: nauc_map_at_1000_max
|
|
value: 14.410594409695182
|
|
- type: nauc_map_at_1000_std
|
|
value: 7.94738583292685
|
|
- type: nauc_map_at_100_diff1
|
|
value: 3.738796209193936
|
|
- type: nauc_map_at_100_max
|
|
value: 14.408029101534694
|
|
- type: nauc_map_at_100_std
|
|
value: 7.979641077687816
|
|
- type: nauc_map_at_10_diff1
|
|
value: 3.334917978089454
|
|
- type: nauc_map_at_10_max
|
|
value: 13.975255289147748
|
|
- type: nauc_map_at_10_std
|
|
value: 7.491959628012161
|
|
- type: nauc_map_at_1_diff1
|
|
value: 75.35066482050009
|
|
- type: nauc_map_at_1_max
|
|
value: 53.573503488571475
|
|
- type: nauc_map_at_1_std
|
|
value: -6.542030594426993
|
|
- type: nauc_map_at_20_diff1
|
|
value: 3.5197129341582083
|
|
- type: nauc_map_at_20_max
|
|
value: 14.159880698006816
|
|
- type: nauc_map_at_20_std
|
|
value: 7.856574384998483
|
|
- type: nauc_map_at_3_diff1
|
|
value: 3.0992333232864064
|
|
- type: nauc_map_at_3_max
|
|
value: 12.513959281222112
|
|
- type: nauc_map_at_3_std
|
|
value: 4.352912866014865
|
|
- type: nauc_map_at_5_diff1
|
|
value: 3.0351688998572537
|
|
- type: nauc_map_at_5_max
|
|
value: 13.21599457624529
|
|
- type: nauc_map_at_5_std
|
|
value: 6.246882983214777
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 75.23953736361132
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 56.64260717262164
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -4.865932053762276
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 75.24091372816497
|
|
- type: nauc_mrr_at_100_max
|
|
value: 56.64831104504846
|
|
- type: nauc_mrr_at_100_std
|
|
value: -4.850966297943324
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 75.26540178053416
|
|
- type: nauc_mrr_at_10_max
|
|
value: 56.828755673428965
|
|
- type: nauc_mrr_at_10_std
|
|
value: -4.8401126970944635
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 75.35066482050009
|
|
- type: nauc_mrr_at_1_max
|
|
value: 53.573503488571475
|
|
- type: nauc_mrr_at_1_std
|
|
value: -6.542030594426993
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 75.24453050729845
|
|
- type: nauc_mrr_at_20_max
|
|
value: 56.69220588401435
|
|
- type: nauc_mrr_at_20_std
|
|
value: -4.843700730832108
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 74.98411648336175
|
|
- type: nauc_mrr_at_3_max
|
|
value: 56.766537573537114
|
|
- type: nauc_mrr_at_3_std
|
|
value: -4.909712671649337
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 75.20599020991028
|
|
- type: nauc_mrr_at_5_max
|
|
value: 56.64236207782237
|
|
- type: nauc_mrr_at_5_std
|
|
value: -5.208907367513977
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 11.48307079099774
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 20.893326881675176
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 10.43489838692119
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 10.395588735754927
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 20.529573302516912
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 11.252973083654268
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 8.596739352741972
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 18.475863682540673
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 9.175831033463352
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 75.35066482050009
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 53.573503488571475
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -6.542030594426993
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 8.998033972471749
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 18.892085875404522
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 10.3241608901084
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 8.796384949533579
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 16.515261419885274
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 4.081902976576701
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 8.277259464605025
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 17.163053202909527
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 6.652669449704474
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -3.490556596304827
|
|
- type: nauc_precision_at_1000_max
|
|
value: 31.0473259001597
|
|
- type: nauc_precision_at_1000_std
|
|
value: 52.36921397692622
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -6.420747959222489
|
|
- type: nauc_precision_at_100_max
|
|
value: 20.555887056005936
|
|
- type: nauc_precision_at_100_std
|
|
value: 36.119132870798495
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -6.461726057290426
|
|
- type: nauc_precision_at_10_max
|
|
value: 12.161081825341915
|
|
- type: nauc_precision_at_10_std
|
|
value: 17.961318451839993
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 75.35066482050009
|
|
- type: nauc_precision_at_1_max
|
|
value: 53.573503488571475
|
|
- type: nauc_precision_at_1_std
|
|
value: -6.542030594426993
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -7.361461296416161
|
|
- type: nauc_precision_at_20_max
|
|
value: 12.663621261696733
|
|
- type: nauc_precision_at_20_std
|
|
value: 23.312476851670286
|
|
- type: nauc_precision_at_3_diff1
|
|
value: -3.299056912774522
|
|
- type: nauc_precision_at_3_max
|
|
value: 9.85602375812038
|
|
- type: nauc_precision_at_3_std
|
|
value: 6.4962782003155475
|
|
- type: nauc_precision_at_5_diff1
|
|
value: -5.3155827772027795
|
|
- type: nauc_precision_at_5_max
|
|
value: 10.32907751171833
|
|
- type: nauc_precision_at_5_std
|
|
value: 11.384098087196932
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: -3.4905565963043332
|
|
- type: nauc_recall_at_1000_max
|
|
value: 31.04732590016041
|
|
- type: nauc_recall_at_1000_std
|
|
value: 52.36921397692641
|
|
- type: nauc_recall_at_100_diff1
|
|
value: -6.420747959222586
|
|
- type: nauc_recall_at_100_max
|
|
value: 20.55588705600596
|
|
- type: nauc_recall_at_100_std
|
|
value: 36.11913287079825
|
|
- type: nauc_recall_at_10_diff1
|
|
value: -6.461726057290347
|
|
- type: nauc_recall_at_10_max
|
|
value: 12.161081825342022
|
|
- type: nauc_recall_at_10_std
|
|
value: 17.96131845184002
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 75.35066482050009
|
|
- type: nauc_recall_at_1_max
|
|
value: 53.573503488571475
|
|
- type: nauc_recall_at_1_std
|
|
value: -6.542030594426993
|
|
- type: nauc_recall_at_20_diff1
|
|
value: -7.361461296416054
|
|
- type: nauc_recall_at_20_max
|
|
value: 12.66362126169679
|
|
- type: nauc_recall_at_20_std
|
|
value: 23.312476851670382
|
|
- type: nauc_recall_at_3_diff1
|
|
value: -3.2990569127745886
|
|
- type: nauc_recall_at_3_max
|
|
value: 9.856023758120296
|
|
- type: nauc_recall_at_3_std
|
|
value: 6.496278200315444
|
|
- type: nauc_recall_at_5_diff1
|
|
value: -5.315582777202729
|
|
- type: nauc_recall_at_5_max
|
|
value: 10.329077511718229
|
|
- type: nauc_recall_at_5_std
|
|
value: 11.384098087196932
|
|
- type: ndcg_at_1
|
|
value: 87.643
|
|
- type: ndcg_at_10
|
|
value: 76.67399999999999
|
|
- type: ndcg_at_100
|
|
value: 79.462
|
|
- type: ndcg_at_1000
|
|
value: 80.43599999999999
|
|
- type: ndcg_at_20
|
|
value: 77.83
|
|
- type: ndcg_at_3
|
|
value: 72.256
|
|
- type: ndcg_at_5
|
|
value: 74.789
|
|
- type: precision_at_1
|
|
value: 87.643
|
|
- type: precision_at_10
|
|
value: 15.726999999999999
|
|
- type: precision_at_100
|
|
value: 1.791
|
|
- type: precision_at_1000
|
|
value: 0.192
|
|
- type: precision_at_20
|
|
value: 8.236
|
|
- type: precision_at_3
|
|
value: 45.919
|
|
- type: precision_at_5
|
|
value: 29.558
|
|
- type: recall_at_1
|
|
value: 43.822
|
|
- type: recall_at_10
|
|
value: 78.636
|
|
- type: recall_at_100
|
|
value: 89.527
|
|
- type: recall_at_1000
|
|
value: 95.868
|
|
- type: recall_at_20
|
|
value: 82.363
|
|
- type: recall_at_3
|
|
value: 68.879
|
|
- type: recall_at_5
|
|
value: 73.896
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB ImdbClassification
|
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7
|
|
split: test
|
|
type: mteb/imdb
|
|
metrics:
|
|
- type: accuracy
|
|
value: 96.6608
|
|
- type: ap
|
|
value: 95.14657820401189
|
|
- type: ap_weighted
|
|
value: 95.14657820401189
|
|
- type: f1
|
|
value: 96.66029695623422
|
|
- type: f1_weighted
|
|
value: 96.66029695623423
|
|
- type: main_score
|
|
value: 96.6608
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB MSMARCO
|
|
revision: c5a29a104738b98a9e76336939199e264163d4a0
|
|
split: dev
|
|
type: mteb/msmarco
|
|
metrics:
|
|
- type: main_score
|
|
value: 45.217
|
|
- type: map_at_1
|
|
value: 24.728
|
|
- type: map_at_10
|
|
value: 37.933
|
|
- type: map_at_100
|
|
value: 39.074999999999996
|
|
- type: map_at_1000
|
|
value: 39.115
|
|
- type: map_at_20
|
|
value: 38.663
|
|
- type: map_at_3
|
|
value: 33.904
|
|
- type: map_at_5
|
|
value: 36.217
|
|
- type: mrr_at_1
|
|
value: 25.44412607449857
|
|
- type: mrr_at_10
|
|
value: 38.52640196479737
|
|
- type: mrr_at_100
|
|
value: 39.60462889736067
|
|
- type: mrr_at_1000
|
|
value: 39.638904296248526
|
|
- type: mrr_at_20
|
|
value: 39.2234365827559
|
|
- type: mrr_at_3
|
|
value: 34.59646609360076
|
|
- type: mrr_at_5
|
|
value: 36.8801337153773
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 37.645652178132174
|
|
- type: nauc_map_at_1000_max
|
|
value: 9.953357023361367
|
|
- type: nauc_map_at_1000_std
|
|
value: -20.800238036721503
|
|
- type: nauc_map_at_100_diff1
|
|
value: 37.643073495974555
|
|
- type: nauc_map_at_100_max
|
|
value: 9.95921239641703
|
|
- type: nauc_map_at_100_std
|
|
value: -20.76517765535793
|
|
- type: nauc_map_at_10_diff1
|
|
value: 37.44380763335014
|
|
- type: nauc_map_at_10_max
|
|
value: 9.917273043055342
|
|
- type: nauc_map_at_10_std
|
|
value: -21.467951225710898
|
|
- type: nauc_map_at_1_diff1
|
|
value: 41.02118887981969
|
|
- type: nauc_map_at_1_max
|
|
value: 8.301113449711778
|
|
- type: nauc_map_at_1_std
|
|
value: -19.436814224415027
|
|
- type: nauc_map_at_20_diff1
|
|
value: 37.58156586490493
|
|
- type: nauc_map_at_20_max
|
|
value: 9.972927967610659
|
|
- type: nauc_map_at_20_std
|
|
value: -20.951374218839387
|
|
- type: nauc_map_at_3_diff1
|
|
value: 37.67246795684178
|
|
- type: nauc_map_at_3_max
|
|
value: 9.307031378909478
|
|
- type: nauc_map_at_3_std
|
|
value: -21.77026217965021
|
|
- type: nauc_map_at_5_diff1
|
|
value: 37.39086482095963
|
|
- type: nauc_map_at_5_max
|
|
value: 9.732739107368566
|
|
- type: nauc_map_at_5_std
|
|
value: -21.8424296893692
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 37.36666719603192
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 9.79040465289953
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -20.590147245965568
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 37.36560296629318
|
|
- type: nauc_mrr_at_100_max
|
|
value: 9.798113710672162
|
|
- type: nauc_mrr_at_100_std
|
|
value: -20.556791838504292
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 37.19257605840734
|
|
- type: nauc_mrr_at_10_max
|
|
value: 9.749429811638063
|
|
- type: nauc_mrr_at_10_std
|
|
value: -21.206407664327276
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 40.98478651095172
|
|
- type: nauc_mrr_at_1_max
|
|
value: 8.173841799119707
|
|
- type: nauc_mrr_at_1_std
|
|
value: -19.530027987868017
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 37.29973172861245
|
|
- type: nauc_mrr_at_20_max
|
|
value: 9.815127660001345
|
|
- type: nauc_mrr_at_20_std
|
|
value: -20.700860112175928
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 37.282848009425734
|
|
- type: nauc_mrr_at_3_max
|
|
value: 9.172741713108193
|
|
- type: nauc_mrr_at_3_std
|
|
value: -21.563630513502996
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 37.08609827303586
|
|
- type: nauc_mrr_at_5_max
|
|
value: 9.604643424273284
|
|
- type: nauc_mrr_at_5_std
|
|
value: -21.580110806494094
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 37.086587020218545
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 10.696860688467472
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -19.50989939916873
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 37.03794531268128
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 10.940820719182339
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -18.28651832370893
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 36.21062857920633
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 10.845172882571733
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -21.454301679510106
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 40.98478651095172
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 8.173841799119707
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -19.530027987868017
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 36.583262733100526
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 11.10492720898974
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -19.41753284137609
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 36.57271365035382
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 9.56073433062999
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -22.324263670932915
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 36.09419372820154
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 10.357384992631271
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -22.389578276324894
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -2.7435338714030597
|
|
- type: nauc_precision_at_1000_max
|
|
value: 4.302274933383809
|
|
- type: nauc_precision_at_1000_std
|
|
value: 8.456846348638948
|
|
- type: nauc_precision_at_100_diff1
|
|
value: 15.149466332615983
|
|
- type: nauc_precision_at_100_max
|
|
value: 12.501013731673163
|
|
- type: nauc_precision_at_100_std
|
|
value: 15.909667509021785
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 28.699788688314214
|
|
- type: nauc_precision_at_10_max
|
|
value: 13.024586051842347
|
|
- type: nauc_precision_at_10_std
|
|
value: -19.197658937078703
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 40.98478651095172
|
|
- type: nauc_precision_at_1_max
|
|
value: 8.173841799119707
|
|
- type: nauc_precision_at_1_std
|
|
value: -19.530027987868017
|
|
- type: nauc_precision_at_20_diff1
|
|
value: 26.519292942353395
|
|
- type: nauc_precision_at_20_max
|
|
value: 14.389979272056438
|
|
- type: nauc_precision_at_20_std
|
|
value: -7.030956994938155
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 32.87913492278213
|
|
- type: nauc_precision_at_3_max
|
|
value: 9.673660161387776
|
|
- type: nauc_precision_at_3_std
|
|
value: -23.905612656592172
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 30.903850113238597
|
|
- type: nauc_precision_at_5_max
|
|
value: 11.482375434154898
|
|
- type: nauc_precision_at_5_std
|
|
value: -23.828657095254247
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 35.80765639589219
|
|
- type: nauc_recall_at_1000_max
|
|
value: 50.94532805969448
|
|
- type: nauc_recall_at_1000_std
|
|
value: 66.79910877083275
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 34.96182828311028
|
|
- type: nauc_recall_at_100_max
|
|
value: 21.729699631790556
|
|
- type: nauc_recall_at_100_std
|
|
value: 23.509439011686474
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 31.88371369567137
|
|
- type: nauc_recall_at_10_max
|
|
value: 14.425389702697073
|
|
- type: nauc_recall_at_10_std
|
|
value: -20.95578001880924
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 41.02118887981969
|
|
- type: nauc_recall_at_1_max
|
|
value: 8.301113449711778
|
|
- type: nauc_recall_at_1_std
|
|
value: -19.436814224415027
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 32.42718780622455
|
|
- type: nauc_recall_at_20_max
|
|
value: 16.90686126329399
|
|
- type: nauc_recall_at_20_std
|
|
value: -9.38158227016737
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 33.68966646043966
|
|
- type: nauc_recall_at_3_max
|
|
value: 10.336277419708532
|
|
- type: nauc_recall_at_3_std
|
|
value: -23.80165869168538
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 32.26258807452426
|
|
- type: nauc_recall_at_5_max
|
|
value: 12.303713005399935
|
|
- type: nauc_recall_at_5_std
|
|
value: -23.87721891164968
|
|
- type: ndcg_at_1
|
|
value: 25.444
|
|
- type: ndcg_at_10
|
|
value: 45.217
|
|
- type: ndcg_at_100
|
|
value: 50.575
|
|
- type: ndcg_at_1000
|
|
value: 51.519999999999996
|
|
- type: ndcg_at_20
|
|
value: 47.786
|
|
- type: ndcg_at_3
|
|
value: 37.067
|
|
- type: ndcg_at_5
|
|
value: 41.184
|
|
- type: precision_at_1
|
|
value: 25.444
|
|
- type: precision_at_10
|
|
value: 7.07
|
|
- type: precision_at_100
|
|
value: 0.9730000000000001
|
|
- type: precision_at_1000
|
|
value: 0.106
|
|
- type: precision_at_20
|
|
value: 4.072
|
|
- type: precision_at_3
|
|
value: 15.754999999999999
|
|
- type: precision_at_5
|
|
value: 11.544
|
|
- type: recall_at_1
|
|
value: 24.728
|
|
- type: recall_at_10
|
|
value: 67.607
|
|
- type: recall_at_100
|
|
value: 92.094
|
|
- type: recall_at_1000
|
|
value: 99.165
|
|
- type: recall_at_20
|
|
value: 77.529
|
|
- type: recall_at_3
|
|
value: 45.535
|
|
- type: recall_at_5
|
|
value: 55.394
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: en
|
|
name: MTEB MTOPDomainClassification (en)
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
|
|
split: test
|
|
type: mteb/mtop_domain
|
|
metrics:
|
|
- type: accuracy
|
|
value: 99.01276789785682
|
|
- type: f1
|
|
value: 98.9288649250924
|
|
- type: f1_weighted
|
|
value: 99.01406884928141
|
|
- type: main_score
|
|
value: 99.01276789785682
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: en
|
|
name: MTEB MTOPIntentClassification (en)
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
|
|
split: test
|
|
type: mteb/mtop_intent
|
|
metrics:
|
|
- type: accuracy
|
|
value: 92.78385772913816
|
|
- type: f1
|
|
value: 79.78115704297824
|
|
- type: f1_weighted
|
|
value: 93.90424147486428
|
|
- type: main_score
|
|
value: 92.78385772913816
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: en
|
|
name: MTEB MassiveIntentClassification (en)
|
|
revision: 4672e20407010da34463acc759c162ca9734bca6
|
|
split: test
|
|
type: mteb/amazon_massive_intent
|
|
metrics:
|
|
- type: accuracy
|
|
value: 85.83053127101546
|
|
- type: f1
|
|
value: 82.72036139888232
|
|
- type: f1_weighted
|
|
value: 85.81759723866098
|
|
- type: main_score
|
|
value: 85.83053127101546
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: en
|
|
name: MTEB MassiveScenarioClassification (en)
|
|
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
|
|
split: test
|
|
type: mteb/amazon_massive_scenario
|
|
metrics:
|
|
- type: accuracy
|
|
value: 90.19838601210489
|
|
- type: f1
|
|
value: 89.55260197964978
|
|
- type: f1_weighted
|
|
value: 90.11422965504119
|
|
- type: main_score
|
|
value: 90.19838601210489
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB MedrxivClusteringP2P
|
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73
|
|
split: test
|
|
type: mteb/medrxiv-clustering-p2p
|
|
metrics:
|
|
- type: main_score
|
|
value: 46.866746897607094
|
|
- type: v_measure
|
|
value: 46.866746897607094
|
|
- type: v_measure_std
|
|
value: 1.0966477896919726
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB MedrxivClusteringS2S
|
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663
|
|
split: test
|
|
type: mteb/medrxiv-clustering-s2s
|
|
metrics:
|
|
- type: main_score
|
|
value: 44.6538827415503
|
|
- type: v_measure
|
|
value: 44.6538827415503
|
|
- type: v_measure_std
|
|
value: 1.1649569936599116
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB MindSmallReranking
|
|
revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7
|
|
split: test
|
|
type: mteb/mind_small
|
|
metrics:
|
|
- type: main_score
|
|
value: 33.05449204940555
|
|
- type: map
|
|
value: 33.05449204940555
|
|
- type: mrr
|
|
value: 34.32562058439585
|
|
- type: nAUC_map_diff1
|
|
value: 11.465656013162807
|
|
- type: nAUC_map_max
|
|
value: -20.400088169502308
|
|
- type: nAUC_map_std
|
|
value: -2.638964886362445
|
|
- type: nAUC_mrr_diff1
|
|
value: 10.644290702481207
|
|
- type: nAUC_mrr_max
|
|
value: -15.304687384645769
|
|
- type: nAUC_mrr_std
|
|
value: -0.519919931348978
|
|
task:
|
|
type: Reranking
|
|
- dataset:
|
|
config: default
|
|
name: MTEB NFCorpus
|
|
revision: ec0fa4fe99da2ff19ca1214b7966684033a58814
|
|
split: test
|
|
type: mteb/nfcorpus
|
|
metrics:
|
|
- type: main_score
|
|
value: 41.998000000000005
|
|
- type: map_at_1
|
|
value: 6.907000000000001
|
|
- type: map_at_10
|
|
value: 16.397000000000002
|
|
- type: map_at_100
|
|
value: 21.69
|
|
- type: map_at_1000
|
|
value: 23.652
|
|
- type: map_at_20
|
|
value: 18.629
|
|
- type: map_at_3
|
|
value: 11.969000000000001
|
|
- type: map_at_5
|
|
value: 13.894
|
|
- type: mrr_at_1
|
|
value: 53.25077399380805
|
|
- type: mrr_at_10
|
|
value: 61.8561108653988
|
|
- type: mrr_at_100
|
|
value: 62.42447851935404
|
|
- type: mrr_at_1000
|
|
value: 62.459626424428095
|
|
- type: mrr_at_20
|
|
value: 62.287236389990696
|
|
- type: mrr_at_3
|
|
value: 60.42311661506711
|
|
- type: mrr_at_5
|
|
value: 61.36738906088753
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 17.159461939643844
|
|
- type: nauc_map_at_1000_max
|
|
value: 32.42764938789903
|
|
- type: nauc_map_at_1000_std
|
|
value: 11.039427848422093
|
|
- type: nauc_map_at_100_diff1
|
|
value: 19.089532984187503
|
|
- type: nauc_map_at_100_max
|
|
value: 31.96721085058713
|
|
- type: nauc_map_at_100_std
|
|
value: 6.947468655726444
|
|
- type: nauc_map_at_10_diff1
|
|
value: 25.77255342629802
|
|
- type: nauc_map_at_10_max
|
|
value: 26.163590320961543
|
|
- type: nauc_map_at_10_std
|
|
value: -5.2588093720998375
|
|
- type: nauc_map_at_1_diff1
|
|
value: 46.31602607957798
|
|
- type: nauc_map_at_1_max
|
|
value: 11.807757660801942
|
|
- type: nauc_map_at_1_std
|
|
value: -13.984889089354317
|
|
- type: nauc_map_at_20_diff1
|
|
value: 22.308161130465365
|
|
- type: nauc_map_at_20_max
|
|
value: 29.070587307827722
|
|
- type: nauc_map_at_20_std
|
|
value: -1.0103056620851558
|
|
- type: nauc_map_at_3_diff1
|
|
value: 33.580827849617506
|
|
- type: nauc_map_at_3_max
|
|
value: 17.661630885799042
|
|
- type: nauc_map_at_3_std
|
|
value: -11.463282544041888
|
|
- type: nauc_map_at_5_diff1
|
|
value: 30.32603342696912
|
|
- type: nauc_map_at_5_max
|
|
value: 20.938905485667245
|
|
- type: nauc_map_at_5_std
|
|
value: -10.537086968155755
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 24.45065397805829
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 48.17519860927417
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 30.350767549118903
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 24.444061606534486
|
|
- type: nauc_mrr_at_100_max
|
|
value: 48.1922894212229
|
|
- type: nauc_mrr_at_100_std
|
|
value: 30.379257816584094
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 24.25598717198779
|
|
- type: nauc_mrr_at_10_max
|
|
value: 48.10437607774264
|
|
- type: nauc_mrr_at_10_std
|
|
value: 30.090202482685996
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 26.907595285201264
|
|
- type: nauc_mrr_at_1_max
|
|
value: 44.006974050369955
|
|
- type: nauc_mrr_at_1_std
|
|
value: 26.921001962861062
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 24.462771570553738
|
|
- type: nauc_mrr_at_20_max
|
|
value: 48.264688196799746
|
|
- type: nauc_mrr_at_20_std
|
|
value: 30.498095141265914
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 24.76829388237229
|
|
- type: nauc_mrr_at_3_max
|
|
value: 48.213758704739924
|
|
- type: nauc_mrr_at_3_std
|
|
value: 30.1502853918892
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 24.476494932330247
|
|
- type: nauc_mrr_at_5_max
|
|
value: 47.977250552198804
|
|
- type: nauc_mrr_at_5_std
|
|
value: 29.65248143104835
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 13.055818920426246
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 46.00986444256306
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 29.622662054922085
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 12.260551238228816
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 39.89783048267698
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 23.806961617956613
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 11.002915931619567
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 39.79323759244374
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 23.053072152911046
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 27.560910719974434
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 41.21084046258119
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 26.112891742912893
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 10.085854089024496
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 37.88629173784684
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 23.17664322248358
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 16.58969583405987
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 41.282222954101435
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 21.080670648392747
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 13.893127947909885
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 40.21188015992804
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 21.417443978842652
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -17.227504530334564
|
|
- type: nauc_precision_at_1000_max
|
|
value: 3.798554468439066
|
|
- type: nauc_precision_at_1000_std
|
|
value: 35.73617809452683
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -17.63388230218776
|
|
- type: nauc_precision_at_100_max
|
|
value: 15.079399882407094
|
|
- type: nauc_precision_at_100_std
|
|
value: 41.83698491321226
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -11.850925959645156
|
|
- type: nauc_precision_at_10_max
|
|
value: 35.93283968364352
|
|
- type: nauc_precision_at_10_std
|
|
value: 34.391271855921296
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 27.730860778824823
|
|
- type: nauc_precision_at_1_max
|
|
value: 43.97462471516834
|
|
- type: nauc_precision_at_1_std
|
|
value: 27.491068270978896
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -14.281328840943347
|
|
- type: nauc_precision_at_20_max
|
|
value: 29.469099781759006
|
|
- type: nauc_precision_at_20_std
|
|
value: 38.54703022340941
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 3.486986910413196
|
|
- type: nauc_precision_at_3_max
|
|
value: 41.21107780473768
|
|
- type: nauc_precision_at_3_std
|
|
value: 24.057479124531216
|
|
- type: nauc_precision_at_5_diff1
|
|
value: -3.0623787872866233
|
|
- type: nauc_precision_at_5_max
|
|
value: 37.49266386466702
|
|
- type: nauc_precision_at_5_std
|
|
value: 26.894454268004935
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: -2.446891864334283
|
|
- type: nauc_recall_at_1000_max
|
|
value: 23.867293584643377
|
|
- type: nauc_recall_at_1000_std
|
|
value: 16.34707128224595
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 4.891133690841179
|
|
- type: nauc_recall_at_100_max
|
|
value: 24.56727964996522
|
|
- type: nauc_recall_at_100_std
|
|
value: 9.847212953200797
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 19.211912363585288
|
|
- type: nauc_recall_at_10_max
|
|
value: 24.825344777920737
|
|
- type: nauc_recall_at_10_std
|
|
value: -5.447989195041898
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 46.31602607957798
|
|
- type: nauc_recall_at_1_max
|
|
value: 11.807757660801942
|
|
- type: nauc_recall_at_1_std
|
|
value: -13.984889089354317
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 12.233372054304805
|
|
- type: nauc_recall_at_20_max
|
|
value: 22.284108685207148
|
|
- type: nauc_recall_at_20_std
|
|
value: -4.317138366746209
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 28.394631527225815
|
|
- type: nauc_recall_at_3_max
|
|
value: 15.593864852625462
|
|
- type: nauc_recall_at_3_std
|
|
value: -12.383531804314593
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 24.457441304950343
|
|
- type: nauc_recall_at_5_max
|
|
value: 19.080049396281623
|
|
- type: nauc_recall_at_5_std
|
|
value: -11.879747703626627
|
|
- type: ndcg_at_1
|
|
value: 51.548
|
|
- type: ndcg_at_10
|
|
value: 41.998000000000005
|
|
- type: ndcg_at_100
|
|
value: 39.626
|
|
- type: ndcg_at_1000
|
|
value: 48.707
|
|
- type: ndcg_at_20
|
|
value: 40.181
|
|
- type: ndcg_at_3
|
|
value: 48.06
|
|
- type: ndcg_at_5
|
|
value: 45.829
|
|
- type: precision_at_1
|
|
value: 52.941
|
|
- type: precision_at_10
|
|
value: 31.330999999999996
|
|
- type: precision_at_100
|
|
value: 10.421
|
|
- type: precision_at_1000
|
|
value: 2.428
|
|
- type: precision_at_20
|
|
value: 24.118000000000002
|
|
- type: precision_at_3
|
|
value: 45.408
|
|
- type: precision_at_5
|
|
value: 39.938
|
|
- type: recall_at_1
|
|
value: 6.907000000000001
|
|
- type: recall_at_10
|
|
value: 20.51
|
|
- type: recall_at_100
|
|
value: 40.857
|
|
- type: recall_at_1000
|
|
value: 73.616
|
|
- type: recall_at_20
|
|
value: 26.52
|
|
- type: recall_at_3
|
|
value: 13.267999999999999
|
|
- type: recall_at_5
|
|
value: 16.141
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB NQ
|
|
revision: b774495ed302d8c44a3a7ea25c90dbce03968f31
|
|
split: test
|
|
type: mteb/nq
|
|
metrics:
|
|
- type: main_score
|
|
value: 71.8
|
|
- type: map_at_1
|
|
value: 47.629
|
|
- type: map_at_10
|
|
value: 64.846
|
|
- type: map_at_100
|
|
value: 65.40899999999999
|
|
- type: map_at_1000
|
|
value: 65.416
|
|
- type: map_at_20
|
|
value: 65.239
|
|
- type: map_at_3
|
|
value: 61.185
|
|
- type: map_at_5
|
|
value: 63.583
|
|
- type: mrr_at_1
|
|
value: 53.15758980301275
|
|
- type: mrr_at_10
|
|
value: 67.12880961577366
|
|
- type: mrr_at_100
|
|
value: 67.44006405426018
|
|
- type: mrr_at_1000
|
|
value: 67.44519150402294
|
|
- type: mrr_at_20
|
|
value: 67.34317135515428
|
|
- type: mrr_at_3
|
|
value: 64.5905755117805
|
|
- type: mrr_at_5
|
|
value: 66.24613750482806
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 45.73812106517133
|
|
- type: nauc_map_at_1000_max
|
|
value: 35.21262031755756
|
|
- type: nauc_map_at_1000_std
|
|
value: -5.549443574026027
|
|
- type: nauc_map_at_100_diff1
|
|
value: 45.74254652176879
|
|
- type: nauc_map_at_100_max
|
|
value: 35.22349167515518
|
|
- type: nauc_map_at_100_std
|
|
value: -5.53697496044773
|
|
- type: nauc_map_at_10_diff1
|
|
value: 45.62837128377087
|
|
- type: nauc_map_at_10_max
|
|
value: 35.3261562342222
|
|
- type: nauc_map_at_10_std
|
|
value: -5.761924414031163
|
|
- type: nauc_map_at_1_diff1
|
|
value: 48.69187848570499
|
|
- type: nauc_map_at_1_max
|
|
value: 28.687996096473476
|
|
- type: nauc_map_at_1_std
|
|
value: -7.518605958272523
|
|
- type: nauc_map_at_20_diff1
|
|
value: 45.702303442220035
|
|
- type: nauc_map_at_20_max
|
|
value: 35.30719944705456
|
|
- type: nauc_map_at_20_std
|
|
value: -5.59505654742681
|
|
- type: nauc_map_at_3_diff1
|
|
value: 45.376813726832474
|
|
- type: nauc_map_at_3_max
|
|
value: 34.68452149643597
|
|
- type: nauc_map_at_3_std
|
|
value: -7.329014950379634
|
|
- type: nauc_map_at_5_diff1
|
|
value: 45.29528861989316
|
|
- type: nauc_map_at_5_max
|
|
value: 35.35741440869229
|
|
- type: nauc_map_at_5_std
|
|
value: -6.028788612259288
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 46.11808147912517
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 35.59241850411947
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -3.4072428526109317
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 46.121345545514046
|
|
- type: nauc_mrr_at_100_max
|
|
value: 35.60147795073431
|
|
- type: nauc_mrr_at_100_std
|
|
value: -3.3965322447588826
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 46.0920068210502
|
|
- type: nauc_mrr_at_10_max
|
|
value: 35.79649987854354
|
|
- type: nauc_mrr_at_10_std
|
|
value: -3.339624589368137
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 49.101364605656194
|
|
- type: nauc_mrr_at_1_max
|
|
value: 31.500796071482146
|
|
- type: nauc_mrr_at_1_std
|
|
value: -4.183818500718156
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 46.088076630465594
|
|
- type: nauc_mrr_at_20_max
|
|
value: 35.682131663053205
|
|
- type: nauc_mrr_at_20_std
|
|
value: -3.35939023178519
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 45.47570812708642
|
|
- type: nauc_mrr_at_3_max
|
|
value: 35.741892517632984
|
|
- type: nauc_mrr_at_3_std
|
|
value: -4.135335963822013
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 45.78903474184014
|
|
- type: nauc_mrr_at_5_max
|
|
value: 35.91273593700205
|
|
- type: nauc_mrr_at_5_std
|
|
value: -3.467873421286869
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 45.5056583000012
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 36.34328379251593
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -4.0759698229323345
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 45.61918946477166
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 36.675460335836235
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -3.6795334726235986
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 45.15343994274541
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 37.48139242964657
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -4.287039084554882
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 49.101364605656194
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 31.500796071482146
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -4.183818500718156
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 45.310026313402375
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 37.32177497902133
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -3.8214360391282587
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 44.27064370528994
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 36.380294033571396
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -6.844263370898355
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 44.29933499225583
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 37.46477041822136
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -4.866548530467956
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -14.666553359142306
|
|
- type: nauc_precision_at_1000_max
|
|
value: -0.5599759853201481
|
|
- type: nauc_precision_at_1000_std
|
|
value: 16.8370925526591
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -11.816251306246278
|
|
- type: nauc_precision_at_100_max
|
|
value: 2.969819268208207
|
|
- type: nauc_precision_at_100_std
|
|
value: 18.59422946634747
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 1.2050200086029401
|
|
- type: nauc_precision_at_10_max
|
|
value: 17.59930352911209
|
|
- type: nauc_precision_at_10_std
|
|
value: 13.714495717588985
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 49.101364605656194
|
|
- type: nauc_precision_at_1_max
|
|
value: 31.500796071482146
|
|
- type: nauc_precision_at_1_std
|
|
value: -4.183818500718156
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -5.263476664822757
|
|
- type: nauc_precision_at_20_max
|
|
value: 11.42004823600046
|
|
- type: nauc_precision_at_20_std
|
|
value: 16.510514518664994
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 20.116460379305828
|
|
- type: nauc_precision_at_3_max
|
|
value: 31.32235038301311
|
|
- type: nauc_precision_at_3_std
|
|
value: 2.7486717133871923
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 9.57451645335723
|
|
- type: nauc_precision_at_5_max
|
|
value: 25.28449126580587
|
|
- type: nauc_precision_at_5_std
|
|
value: 9.955736162466767
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: -21.632253065978794
|
|
- type: nauc_recall_at_1000_max
|
|
value: 70.14409090958776
|
|
- type: nauc_recall_at_1000_std
|
|
value: 65.61658090892989
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 51.83161124806711
|
|
- type: nauc_recall_at_100_max
|
|
value: 77.49921361841523
|
|
- type: nauc_recall_at_100_std
|
|
value: 48.352508746719444
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 39.86695231362791
|
|
- type: nauc_recall_at_10_max
|
|
value: 50.12029094799474
|
|
- type: nauc_recall_at_10_std
|
|
value: 0.1650940628131058
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 48.69187848570499
|
|
- type: nauc_recall_at_1_max
|
|
value: 28.687996096473476
|
|
- type: nauc_recall_at_1_std
|
|
value: -7.518605958272523
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 39.14155398061627
|
|
- type: nauc_recall_at_20_max
|
|
value: 56.78559423716229
|
|
- type: nauc_recall_at_20_std
|
|
value: 7.9728224572344075
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 38.69589523432158
|
|
- type: nauc_recall_at_3_max
|
|
value: 39.53271258375579
|
|
- type: nauc_recall_at_3_std
|
|
value: -8.646925065787512
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 37.45922652959002
|
|
- type: nauc_recall_at_5_max
|
|
value: 44.4911958995867
|
|
- type: nauc_recall_at_5_std
|
|
value: -3.5659842556375594
|
|
- type: ndcg_at_1
|
|
value: 53.15800000000001
|
|
- type: ndcg_at_10
|
|
value: 71.8
|
|
- type: ndcg_at_100
|
|
value: 73.85199999999999
|
|
- type: ndcg_at_1000
|
|
value: 74.017
|
|
- type: ndcg_at_20
|
|
value: 72.933
|
|
- type: ndcg_at_3
|
|
value: 65.479
|
|
- type: ndcg_at_5
|
|
value: 69.182
|
|
- type: precision_at_1
|
|
value: 53.15800000000001
|
|
- type: precision_at_10
|
|
value: 10.805
|
|
- type: precision_at_100
|
|
value: 1.2
|
|
- type: precision_at_1000
|
|
value: 0.122
|
|
- type: precision_at_20
|
|
value: 5.694
|
|
- type: precision_at_3
|
|
value: 28.939999999999998
|
|
- type: precision_at_5
|
|
value: 19.641000000000002
|
|
- type: recall_at_1
|
|
value: 47.629
|
|
- type: recall_at_10
|
|
value: 90.204
|
|
- type: recall_at_100
|
|
value: 98.66
|
|
- type: recall_at_1000
|
|
value: 99.874
|
|
- type: recall_at_20
|
|
value: 94.24
|
|
- type: recall_at_3
|
|
value: 74.394
|
|
- type: recall_at_5
|
|
value: 82.711
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB QuoraRetrieval
|
|
revision: e4e08e0b7dbe3c8700f0daef558ff32256715259
|
|
split: test
|
|
type: mteb/quora
|
|
metrics:
|
|
- type: main_score
|
|
value: 90.025
|
|
- type: map_at_1
|
|
value: 72.222
|
|
- type: map_at_10
|
|
value: 86.58500000000001
|
|
- type: map_at_100
|
|
value: 87.176
|
|
- type: map_at_1000
|
|
value: 87.188
|
|
- type: map_at_20
|
|
value: 86.97399999999999
|
|
- type: map_at_3
|
|
value: 83.736
|
|
- type: map_at_5
|
|
value: 85.554
|
|
- type: mrr_at_1
|
|
value: 83.04
|
|
- type: mrr_at_10
|
|
value: 89.05599603174585
|
|
- type: mrr_at_100
|
|
value: 89.12398891419457
|
|
- type: mrr_at_1000
|
|
value: 89.12434072241001
|
|
- type: mrr_at_20
|
|
value: 89.10416280692111
|
|
- type: mrr_at_3
|
|
value: 88.23833333333312
|
|
- type: mrr_at_5
|
|
value: 88.82233333333308
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 78.29348113313218
|
|
- type: nauc_map_at_1000_max
|
|
value: 32.31386754277228
|
|
- type: nauc_map_at_1000_std
|
|
value: -50.47543661484052
|
|
- type: nauc_map_at_100_diff1
|
|
value: 78.29618548618575
|
|
- type: nauc_map_at_100_max
|
|
value: 32.301475680947846
|
|
- type: nauc_map_at_100_std
|
|
value: -50.50303428814228
|
|
- type: nauc_map_at_10_diff1
|
|
value: 78.47383776440803
|
|
- type: nauc_map_at_10_max
|
|
value: 31.839339990133563
|
|
- type: nauc_map_at_10_std
|
|
value: -52.832713555976
|
|
- type: nauc_map_at_1_diff1
|
|
value: 82.46330147467418
|
|
- type: nauc_map_at_1_max
|
|
value: 23.497664918373538
|
|
- type: nauc_map_at_1_std
|
|
value: -43.824657665520704
|
|
- type: nauc_map_at_20_diff1
|
|
value: 78.34772176474422
|
|
- type: nauc_map_at_20_max
|
|
value: 32.16495182893947
|
|
- type: nauc_map_at_20_std
|
|
value: -51.503292726558605
|
|
- type: nauc_map_at_3_diff1
|
|
value: 79.07823813069432
|
|
- type: nauc_map_at_3_max
|
|
value: 29.395911687513976
|
|
- type: nauc_map_at_3_std
|
|
value: -54.16377546873304
|
|
- type: nauc_map_at_5_diff1
|
|
value: 78.73076619520454
|
|
- type: nauc_map_at_5_max
|
|
value: 30.700453118585237
|
|
- type: nauc_map_at_5_std
|
|
value: -54.130514177664054
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 79.04736184471865
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 34.43004593837643
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -46.137269068195316
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 79.04698704288086
|
|
- type: nauc_mrr_at_100_max
|
|
value: 34.4305553741175
|
|
- type: nauc_mrr_at_100_std
|
|
value: -46.13786687786434
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 79.04490677485934
|
|
- type: nauc_mrr_at_10_max
|
|
value: 34.38170181522227
|
|
- type: nauc_mrr_at_10_std
|
|
value: -46.38129875681807
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 79.87159215719124
|
|
- type: nauc_mrr_at_1_max
|
|
value: 34.05882339253136
|
|
- type: nauc_mrr_at_1_std
|
|
value: -43.56093395137571
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 79.04384174535653
|
|
- type: nauc_mrr_at_20_max
|
|
value: 34.442136494675005
|
|
- type: nauc_mrr_at_20_std
|
|
value: -46.205458519638654
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 78.78154519155487
|
|
- type: nauc_mrr_at_3_max
|
|
value: 34.74995000500305
|
|
- type: nauc_mrr_at_3_std
|
|
value: -46.36264203155416
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 79.02631187177
|
|
- type: nauc_mrr_at_5_max
|
|
value: 34.538698249632205
|
|
- type: nauc_mrr_at_5_std
|
|
value: -46.468881576157465
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 78.25260097014645
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 33.68584498704271
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -48.44716779494868
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 78.25115412256716
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 33.63652663447088
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -48.489243909024715
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 78.23875101557334
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 32.65217430043823
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -52.57770468845309
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 79.87159215719124
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 34.05882339253136
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -43.56093395137571
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 78.23478552311765
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 33.30691737901109
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -50.78412614854527
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 77.66134485470224
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 32.19504710373125
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -52.01636728550155
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 78.04734137324255
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 31.94593625591248
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -53.02169800690546
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -45.771948123542636
|
|
- type: nauc_precision_at_1000_max
|
|
value: -5.182406190477681
|
|
- type: nauc_precision_at_1000_std
|
|
value: 41.14460438707817
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -45.64767154261461
|
|
- type: nauc_precision_at_100_max
|
|
value: -5.046308286851713
|
|
- type: nauc_precision_at_100_std
|
|
value: 41.07186716587844
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -42.26779562305825
|
|
- type: nauc_precision_at_10_max
|
|
value: -1.1264852893323076
|
|
- type: nauc_precision_at_10_std
|
|
value: 27.62275729822392
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 79.87159215719124
|
|
- type: nauc_precision_at_1_max
|
|
value: 34.05882339253136
|
|
- type: nauc_precision_at_1_std
|
|
value: -43.56093395137571
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -44.24293221128388
|
|
- type: nauc_precision_at_20_max
|
|
value: -3.1345628837361867
|
|
- type: nauc_precision_at_20_std
|
|
value: 34.23625492740366
|
|
- type: nauc_precision_at_3_diff1
|
|
value: -24.925251389823348
|
|
- type: nauc_precision_at_3_max
|
|
value: 6.622188833369412
|
|
- type: nauc_precision_at_3_std
|
|
value: 6.424741786858512
|
|
- type: nauc_precision_at_5_diff1
|
|
value: -36.1407949990387
|
|
- type: nauc_precision_at_5_max
|
|
value: 1.7533948968374462
|
|
- type: nauc_precision_at_5_std
|
|
value: 17.914083278982634
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 52.26815466244496
|
|
- type: nauc_recall_at_1000_max
|
|
value: 69.73611104239443
|
|
- type: nauc_recall_at_1000_std
|
|
value: 73.18969965863008
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 70.80557513785271
|
|
- type: nauc_recall_at_100_max
|
|
value: 33.333440086544556
|
|
- type: nauc_recall_at_100_std
|
|
value: -38.75992366905504
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 74.45948457438163
|
|
- type: nauc_recall_at_10_max
|
|
value: 26.64948512428989
|
|
- type: nauc_recall_at_10_std
|
|
value: -82.90334292052363
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 82.46330147467418
|
|
- type: nauc_recall_at_1_max
|
|
value: 23.497664918373538
|
|
- type: nauc_recall_at_1_std
|
|
value: -43.824657665520704
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 73.80140280887753
|
|
- type: nauc_recall_at_20_max
|
|
value: 30.361616426734965
|
|
- type: nauc_recall_at_20_std
|
|
value: -81.1418804447414
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 75.19854736087834
|
|
- type: nauc_recall_at_3_max
|
|
value: 26.12298005045584
|
|
- type: nauc_recall_at_3_std
|
|
value: -63.42583714745169
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 74.16423451950358
|
|
- type: nauc_recall_at_5_max
|
|
value: 25.552390331018987
|
|
- type: nauc_recall_at_5_std
|
|
value: -71.15891947773912
|
|
- type: ndcg_at_1
|
|
value: 83.04
|
|
- type: ndcg_at_10
|
|
value: 90.025
|
|
- type: ndcg_at_100
|
|
value: 91.006
|
|
- type: ndcg_at_1000
|
|
value: 91.061
|
|
- type: ndcg_at_20
|
|
value: 90.556
|
|
- type: ndcg_at_3
|
|
value: 87.493
|
|
- type: ndcg_at_5
|
|
value: 88.955
|
|
- type: precision_at_1
|
|
value: 83.04
|
|
- type: precision_at_10
|
|
value: 13.667000000000002
|
|
- type: precision_at_100
|
|
value: 1.542
|
|
- type: precision_at_1000
|
|
value: 0.157
|
|
- type: precision_at_20
|
|
value: 7.221
|
|
- type: precision_at_3
|
|
value: 38.433
|
|
- type: precision_at_5
|
|
value: 25.228
|
|
- type: recall_at_1
|
|
value: 72.222
|
|
- type: recall_at_10
|
|
value: 96.604
|
|
- type: recall_at_100
|
|
value: 99.786
|
|
- type: recall_at_1000
|
|
value: 99.996
|
|
- type: recall_at_20
|
|
value: 98.253
|
|
- type: recall_at_3
|
|
value: 89.276
|
|
- type: recall_at_5
|
|
value: 93.46
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB RedditClustering
|
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb
|
|
split: test
|
|
type: mteb/reddit-clustering
|
|
metrics:
|
|
- type: main_score
|
|
value: 72.86492101891123
|
|
- type: v_measure
|
|
value: 72.86492101891123
|
|
- type: v_measure_std
|
|
value: 2.778711445144635
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB RedditClusteringP2P
|
|
revision: 385e3cb46b4cfa89021f56c4380204149d0efe33
|
|
split: test
|
|
type: mteb/reddit-clustering-p2p
|
|
metrics:
|
|
- type: main_score
|
|
value: 75.27316726548479
|
|
- type: v_measure
|
|
value: 75.27316726548479
|
|
- type: v_measure_std
|
|
value: 8.87871936725338
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB SCIDOCS
|
|
revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88
|
|
split: test
|
|
type: mteb/scidocs
|
|
metrics:
|
|
- type: main_score
|
|
value: 26.638
|
|
- type: map_at_1
|
|
value: 6.128
|
|
- type: map_at_10
|
|
value: 16.472
|
|
- type: map_at_100
|
|
value: 19.522000000000002
|
|
- type: map_at_1000
|
|
value: 19.898
|
|
- type: map_at_20
|
|
value: 18.098
|
|
- type: map_at_3
|
|
value: 11.283
|
|
- type: map_at_5
|
|
value: 13.771
|
|
- type: mrr_at_1
|
|
value: 30.2
|
|
- type: mrr_at_10
|
|
value: 42.621150793650735
|
|
- type: mrr_at_100
|
|
value: 43.740858712021954
|
|
- type: mrr_at_1000
|
|
value: 43.762699500220904
|
|
- type: mrr_at_20
|
|
value: 43.383639927753634
|
|
- type: mrr_at_3
|
|
value: 38.83333333333331
|
|
- type: mrr_at_5
|
|
value: 41.14833333333326
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 13.13534664124808
|
|
- type: nauc_map_at_1000_max
|
|
value: 29.346654566149795
|
|
- type: nauc_map_at_1000_std
|
|
value: 18.08121186982413
|
|
- type: nauc_map_at_100_diff1
|
|
value: 13.098072728041538
|
|
- type: nauc_map_at_100_max
|
|
value: 29.299084480697523
|
|
- type: nauc_map_at_100_std
|
|
value: 17.961620202918464
|
|
- type: nauc_map_at_10_diff1
|
|
value: 14.001743720394682
|
|
- type: nauc_map_at_10_max
|
|
value: 28.04128290996403
|
|
- type: nauc_map_at_10_std
|
|
value: 13.744481555974716
|
|
- type: nauc_map_at_1_diff1
|
|
value: 22.1926640424872
|
|
- type: nauc_map_at_1_max
|
|
value: 21.32609279586034
|
|
- type: nauc_map_at_1_std
|
|
value: 6.566596302915438
|
|
- type: nauc_map_at_20_diff1
|
|
value: 13.57313142419664
|
|
- type: nauc_map_at_20_max
|
|
value: 28.93840146319476
|
|
- type: nauc_map_at_20_std
|
|
value: 16.50869367365676
|
|
- type: nauc_map_at_3_diff1
|
|
value: 17.707700541948462
|
|
- type: nauc_map_at_3_max
|
|
value: 26.058174051376238
|
|
- type: nauc_map_at_3_std
|
|
value: 9.943924560735267
|
|
- type: nauc_map_at_5_diff1
|
|
value: 17.11844492157723
|
|
- type: nauc_map_at_5_max
|
|
value: 27.865247403049388
|
|
- type: nauc_map_at_5_std
|
|
value: 11.372588172121546
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 21.11248719936198
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 26.734172102201466
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 11.766121765437228
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 21.107109982277702
|
|
- type: nauc_mrr_at_100_max
|
|
value: 26.741616065723267
|
|
- type: nauc_mrr_at_100_std
|
|
value: 11.789802686224208
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 20.74108639793207
|
|
- type: nauc_mrr_at_10_max
|
|
value: 26.920838463358333
|
|
- type: nauc_mrr_at_10_std
|
|
value: 11.849217361926522
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 22.177437860573356
|
|
- type: nauc_mrr_at_1_max
|
|
value: 21.88074521417754
|
|
- type: nauc_mrr_at_1_std
|
|
value: 6.776011900101789
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 21.126633710175994
|
|
- type: nauc_mrr_at_20_max
|
|
value: 26.860736480370974
|
|
- type: nauc_mrr_at_20_std
|
|
value: 11.815411633726338
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 21.689245200066466
|
|
- type: nauc_mrr_at_3_max
|
|
value: 26.187305092831625
|
|
- type: nauc_mrr_at_3_std
|
|
value: 10.895380313134332
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 20.898811082479778
|
|
- type: nauc_mrr_at_5_max
|
|
value: 26.939217247104036
|
|
- type: nauc_mrr_at_5_std
|
|
value: 11.77832949822472
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 13.251184947898546
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 30.879594164526146
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 23.125206047366625
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 12.549100649053676
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 30.634680845419123
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 23.296226055422984
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 14.475144549294322
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 29.450349815417336
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 15.94068314781612
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 22.177437860573356
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 21.88074521417754
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 6.776011900101789
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 14.173669585802266
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 30.475890854725
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 19.863898148221704
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 18.93971261196868
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 27.3707298720736
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 11.439810510051224
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 17.89535958094687
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 29.272740466638425
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 13.402467626635909
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -3.811547048784123
|
|
- type: nauc_precision_at_1000_max
|
|
value: 22.55165337197117
|
|
- type: nauc_precision_at_1000_std
|
|
value: 35.98524999650108
|
|
- type: nauc_precision_at_100_diff1
|
|
value: 0.6474234774922896
|
|
- type: nauc_precision_at_100_max
|
|
value: 25.06920726527032
|
|
- type: nauc_precision_at_100_std
|
|
value: 32.31439698982313
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 7.943127218139508
|
|
- type: nauc_precision_at_10_max
|
|
value: 28.571937636787197
|
|
- type: nauc_precision_at_10_std
|
|
value: 18.8472620918488
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 22.177437860573356
|
|
- type: nauc_precision_at_1_max
|
|
value: 21.88074521417754
|
|
- type: nauc_precision_at_1_std
|
|
value: 6.776011900101789
|
|
- type: nauc_precision_at_20_diff1
|
|
value: 6.981574259607366
|
|
- type: nauc_precision_at_20_max
|
|
value: 28.986094397038727
|
|
- type: nauc_precision_at_20_std
|
|
value: 25.83129974001146
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 17.197490724039355
|
|
- type: nauc_precision_at_3_max
|
|
value: 29.17569320583099
|
|
- type: nauc_precision_at_3_std
|
|
value: 13.430554945991846
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 14.952364330739362
|
|
- type: nauc_precision_at_5_max
|
|
value: 31.053243354846977
|
|
- type: nauc_precision_at_5_std
|
|
value: 15.856312752807822
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: -4.8224253128926975
|
|
- type: nauc_recall_at_1000_max
|
|
value: 21.3989024429911
|
|
- type: nauc_recall_at_1000_std
|
|
value: 39.152234275603604
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 0.11936808422867201
|
|
- type: nauc_recall_at_100_max
|
|
value: 24.261739241957823
|
|
- type: nauc_recall_at_100_std
|
|
value: 32.62984573938928
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 7.851256165018388
|
|
- type: nauc_recall_at_10_max
|
|
value: 27.936406600938746
|
|
- type: nauc_recall_at_10_std
|
|
value: 18.683634320636113
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 22.1926640424872
|
|
- type: nauc_recall_at_1_max
|
|
value: 21.32609279586034
|
|
- type: nauc_recall_at_1_std
|
|
value: 6.566596302915438
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 6.8107211705182165
|
|
- type: nauc_recall_at_20_max
|
|
value: 28.286284094687787
|
|
- type: nauc_recall_at_20_std
|
|
value: 25.932013268120862
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 17.04156818427151
|
|
- type: nauc_recall_at_3_max
|
|
value: 28.645439108719216
|
|
- type: nauc_recall_at_3_std
|
|
value: 13.346047828494411
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 14.906284329771822
|
|
- type: nauc_recall_at_5_max
|
|
value: 30.58628602415921
|
|
- type: nauc_recall_at_5_std
|
|
value: 15.755157478191755
|
|
- type: ndcg_at_1
|
|
value: 30.2
|
|
- type: ndcg_at_10
|
|
value: 26.638
|
|
- type: ndcg_at_100
|
|
value: 37.135
|
|
- type: ndcg_at_1000
|
|
value: 42.576
|
|
- type: ndcg_at_20
|
|
value: 30.75
|
|
- type: ndcg_at_3
|
|
value: 24.675
|
|
- type: ndcg_at_5
|
|
value: 21.836
|
|
- type: precision_at_1
|
|
value: 30.2
|
|
- type: precision_at_10
|
|
value: 14.06
|
|
- type: precision_at_100
|
|
value: 2.904
|
|
- type: precision_at_1000
|
|
value: 0.42
|
|
- type: precision_at_20
|
|
value: 9.4
|
|
- type: precision_at_3
|
|
value: 23.233
|
|
- type: precision_at_5
|
|
value: 19.439999999999998
|
|
- type: recall_at_1
|
|
value: 6.128
|
|
- type: recall_at_10
|
|
value: 28.471999999999998
|
|
- type: recall_at_100
|
|
value: 58.952000000000005
|
|
- type: recall_at_1000
|
|
value: 85.137
|
|
- type: recall_at_20
|
|
value: 38.17
|
|
- type: recall_at_3
|
|
value: 14.127999999999998
|
|
- type: recall_at_5
|
|
value: 19.673
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB SICK-R
|
|
revision: 20a6d6f312dd54037fe07a32d58e5e168867909d
|
|
split: test
|
|
type: mteb/sickr-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 86.86608529160739
|
|
- type: cosine_spearman
|
|
value: 82.88625166203383
|
|
- type: euclidean_pearson
|
|
value: 84.15494418856142
|
|
- type: euclidean_spearman
|
|
value: 82.88449294676421
|
|
- type: main_score
|
|
value: 82.88625166203383
|
|
- type: manhattan_pearson
|
|
value: 84.39068623474428
|
|
- type: manhattan_spearman
|
|
value: 82.88065412169463
|
|
- type: pearson
|
|
value: 86.86608529160739
|
|
- type: spearman
|
|
value: 82.88625166203383
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB STS12
|
|
revision: a0d554a64d88156834ff5ae9920b964011b16384
|
|
split: test
|
|
type: mteb/sts12-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 87.0445014940449
|
|
- type: cosine_spearman
|
|
value: 80.0880365116599
|
|
- type: euclidean_pearson
|
|
value: 83.80250772928852
|
|
- type: euclidean_spearman
|
|
value: 80.0892465260778
|
|
- type: main_score
|
|
value: 80.0880365116599
|
|
- type: manhattan_pearson
|
|
value: 83.96793981929336
|
|
- type: manhattan_spearman
|
|
value: 80.24881789268238
|
|
- type: pearson
|
|
value: 87.0445014940449
|
|
- type: spearman
|
|
value: 80.0880365116599
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB STS13
|
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca
|
|
split: test
|
|
type: mteb/sts13-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 89.33900828959968
|
|
- type: cosine_spearman
|
|
value: 89.68256358526733
|
|
- type: euclidean_pearson
|
|
value: 89.29188708262265
|
|
- type: euclidean_spearman
|
|
value: 89.68204344658601
|
|
- type: main_score
|
|
value: 89.68256358526733
|
|
- type: manhattan_pearson
|
|
value: 89.13996588193149
|
|
- type: manhattan_spearman
|
|
value: 89.61372804425623
|
|
- type: pearson
|
|
value: 89.33900828959968
|
|
- type: spearman
|
|
value: 89.68256358526733
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB STS14
|
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375
|
|
split: test
|
|
type: mteb/sts14-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 86.42029843639123
|
|
- type: cosine_spearman
|
|
value: 85.0707889220723
|
|
- type: euclidean_pearson
|
|
value: 85.75114239552562
|
|
- type: euclidean_spearman
|
|
value: 85.06858160270725
|
|
- type: main_score
|
|
value: 85.0707889220723
|
|
- type: manhattan_pearson
|
|
value: 85.86461900459038
|
|
- type: manhattan_spearman
|
|
value: 85.28671103475605
|
|
- type: pearson
|
|
value: 86.42029843639123
|
|
- type: spearman
|
|
value: 85.0707889220723
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB STS15
|
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3
|
|
split: test
|
|
type: mteb/sts15-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 88.3660081271444
|
|
- type: cosine_spearman
|
|
value: 89.39375083609528
|
|
- type: euclidean_pearson
|
|
value: 89.21818482894895
|
|
- type: euclidean_spearman
|
|
value: 89.39361588875443
|
|
- type: main_score
|
|
value: 89.39375083609528
|
|
- type: manhattan_pearson
|
|
value: 89.53535068014057
|
|
- type: manhattan_spearman
|
|
value: 89.81077130567752
|
|
- type: pearson
|
|
value: 88.3660081271444
|
|
- type: spearman
|
|
value: 89.39375083609528
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB STS16
|
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513
|
|
split: test
|
|
type: mteb/sts16-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 85.60708247171874
|
|
- type: cosine_spearman
|
|
value: 87.15234952832193
|
|
- type: euclidean_pearson
|
|
value: 86.21743555548137
|
|
- type: euclidean_spearman
|
|
value: 87.14450217418016
|
|
- type: main_score
|
|
value: 87.15234952832193
|
|
- type: manhattan_pearson
|
|
value: 86.2467748746084
|
|
- type: manhattan_spearman
|
|
value: 87.2197479717654
|
|
- type: pearson
|
|
value: 85.60708247171874
|
|
- type: spearman
|
|
value: 87.15234952832193
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: en-en
|
|
name: MTEB STS17 (en-en)
|
|
revision: faeb762787bd10488a50c8b5be4a3b82e411949c
|
|
split: test
|
|
type: mteb/sts17-crosslingual-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 91.25898556808458
|
|
- type: cosine_spearman
|
|
value: 91.35372390581641
|
|
- type: euclidean_pearson
|
|
value: 91.319520321348
|
|
- type: euclidean_spearman
|
|
value: 91.30821135416925
|
|
- type: main_score
|
|
value: 91.35372390581641
|
|
- type: manhattan_pearson
|
|
value: 91.14800959939069
|
|
- type: manhattan_spearman
|
|
value: 91.09775424245629
|
|
- type: pearson
|
|
value: 91.25898556808458
|
|
- type: spearman
|
|
value: 91.35372390581641
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: en
|
|
name: MTEB STS22 (en)
|
|
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
|
|
split: test
|
|
type: mteb/sts22-crosslingual-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 67.61637111515797
|
|
- type: cosine_spearman
|
|
value: 68.10379096526697
|
|
- type: euclidean_pearson
|
|
value: 69.2652309491375
|
|
- type: euclidean_spearman
|
|
value: 68.18436357033228
|
|
- type: main_score
|
|
value: 68.10379096526697
|
|
- type: manhattan_pearson
|
|
value: 69.52531340510775
|
|
- type: manhattan_spearman
|
|
value: 68.17874790391862
|
|
- type: pearson
|
|
value: 67.61637111515797
|
|
- type: spearman
|
|
value: 68.10379096526697
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB STSBenchmark
|
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831
|
|
split: test
|
|
type: mteb/stsbenchmark-sts
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 87.81592853782297
|
|
- type: cosine_spearman
|
|
value: 88.2302550329183
|
|
- type: euclidean_pearson
|
|
value: 88.01165144519526
|
|
- type: euclidean_spearman
|
|
value: 88.23342148890097
|
|
- type: main_score
|
|
value: 88.2302550329183
|
|
- type: manhattan_pearson
|
|
value: 88.148592564938
|
|
- type: manhattan_spearman
|
|
value: 88.49226317320988
|
|
- type: pearson
|
|
value: 87.81592853782297
|
|
- type: spearman
|
|
value: 88.2302550329183
|
|
task:
|
|
type: STS
|
|
- dataset:
|
|
config: default
|
|
name: MTEB SciDocsRR
|
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab
|
|
split: test
|
|
type: mteb/scidocs-reranking
|
|
metrics:
|
|
- type: main_score
|
|
value: 89.196009707431
|
|
- type: map
|
|
value: 89.196009707431
|
|
- type: mrr
|
|
value: 97.07198121413808
|
|
- type: nAUC_map_diff1
|
|
value: -14.066667940115352
|
|
- type: nAUC_map_max
|
|
value: 49.73702475027407
|
|
- type: nAUC_map_std
|
|
value: 64.0986775782592
|
|
- type: nAUC_mrr_diff1
|
|
value: 21.96846389417319
|
|
- type: nAUC_mrr_max
|
|
value: 86.38341077184032
|
|
- type: nAUC_mrr_std
|
|
value: 75.38945014727746
|
|
task:
|
|
type: Reranking
|
|
- dataset:
|
|
config: default
|
|
name: MTEB SciFact
|
|
revision: 0228b52cf27578f30900b9e5271d331663a030d7
|
|
split: test
|
|
type: mteb/scifact
|
|
metrics:
|
|
- type: main_score
|
|
value: 80.08999999999999
|
|
- type: map_at_1
|
|
value: 63.161
|
|
- type: map_at_10
|
|
value: 75.163
|
|
- type: map_at_100
|
|
value: 75.408
|
|
- type: map_at_1000
|
|
value: 75.409
|
|
- type: map_at_20
|
|
value: 75.332
|
|
- type: map_at_3
|
|
value: 71.839
|
|
- type: map_at_5
|
|
value: 74.32600000000001
|
|
- type: mrr_at_1
|
|
value: 66.33333333333333
|
|
- type: mrr_at_10
|
|
value: 75.95978835978836
|
|
- type: mrr_at_100
|
|
value: 76.15647881281473
|
|
- type: mrr_at_1000
|
|
value: 76.15736533763744
|
|
- type: mrr_at_20
|
|
value: 76.08557368557368
|
|
- type: mrr_at_3
|
|
value: 73.55555555555556
|
|
- type: mrr_at_5
|
|
value: 75.4888888888889
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 77.31229383811176
|
|
- type: nauc_map_at_1000_max
|
|
value: 58.848319058605156
|
|
- type: nauc_map_at_1000_std
|
|
value: -14.290090263454985
|
|
- type: nauc_map_at_100_diff1
|
|
value: 77.31325400213969
|
|
- type: nauc_map_at_100_max
|
|
value: 58.848885054155275
|
|
- type: nauc_map_at_100_std
|
|
value: -14.285806618869273
|
|
- type: nauc_map_at_10_diff1
|
|
value: 77.1806705504232
|
|
- type: nauc_map_at_10_max
|
|
value: 59.02905805134415
|
|
- type: nauc_map_at_10_std
|
|
value: -14.132954900037467
|
|
- type: nauc_map_at_1_diff1
|
|
value: 81.03932970557837
|
|
- type: nauc_map_at_1_max
|
|
value: 49.02073230264529
|
|
- type: nauc_map_at_1_std
|
|
value: -22.977452975845512
|
|
- type: nauc_map_at_20_diff1
|
|
value: 77.22581364818562
|
|
- type: nauc_map_at_20_max
|
|
value: 58.90740400399768
|
|
- type: nauc_map_at_20_std
|
|
value: -14.245079150986745
|
|
- type: nauc_map_at_3_diff1
|
|
value: 76.99793243255563
|
|
- type: nauc_map_at_3_max
|
|
value: 54.9930733886623
|
|
- type: nauc_map_at_3_std
|
|
value: -19.297708446082407
|
|
- type: nauc_map_at_5_diff1
|
|
value: 77.1671608360295
|
|
- type: nauc_map_at_5_max
|
|
value: 57.27757489519526
|
|
- type: nauc_map_at_5_std
|
|
value: -15.446338357667708
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 77.4806080821202
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 60.9213776129792
|
|
- type: nauc_mrr_at_1000_std
|
|
value: -12.139599632228343
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 77.48158073865281
|
|
- type: nauc_mrr_at_100_max
|
|
value: 60.9218657185361
|
|
- type: nauc_mrr_at_100_std
|
|
value: -12.13532070453677
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 77.32428546014407
|
|
- type: nauc_mrr_at_10_max
|
|
value: 61.018407010343466
|
|
- type: nauc_mrr_at_10_std
|
|
value: -12.143193773309347
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 80.99806778887115
|
|
- type: nauc_mrr_at_1_max
|
|
value: 59.17855969530095
|
|
- type: nauc_mrr_at_1_std
|
|
value: -12.30545640831458
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 77.3811067653992
|
|
- type: nauc_mrr_at_20_max
|
|
value: 60.9648880366335
|
|
- type: nauc_mrr_at_20_std
|
|
value: -12.124066076541853
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 77.31304316321959
|
|
- type: nauc_mrr_at_3_max
|
|
value: 60.75536766404163
|
|
- type: nauc_mrr_at_3_std
|
|
value: -12.997876030849623
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 77.12952864141742
|
|
- type: nauc_mrr_at_5_max
|
|
value: 60.995943754968685
|
|
- type: nauc_mrr_at_5_std
|
|
value: -11.353447465605694
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 76.81788665683746
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 60.35947755262391
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: -12.884942372460362
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 76.87388230365198
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 60.38813162962434
|
|
- type: nauc_ndcg_at_100_std
|
|
value: -12.64384717800478
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 75.87713506026317
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 61.39356554675667
|
|
- type: nauc_ndcg_at_10_std
|
|
value: -12.144227584144218
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 80.99806778887115
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 59.17855969530095
|
|
- type: nauc_ndcg_at_1_std
|
|
value: -12.30545640831458
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 76.09913944506627
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 61.01644448834147
|
|
- type: nauc_ndcg_at_20_std
|
|
value: -12.456209267623857
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 75.52717946614608
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 58.96433090721983
|
|
- type: nauc_ndcg_at_3_std
|
|
value: -15.849280494339556
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 75.69026981016921
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 58.924044405851326
|
|
- type: nauc_ndcg_at_5_std
|
|
value: -13.182728827923107
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -31.634022001609914
|
|
- type: nauc_precision_at_1000_max
|
|
value: 31.46271490784504
|
|
- type: nauc_precision_at_1000_std
|
|
value: 60.44801276891442
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -29.722363469948103
|
|
- type: nauc_precision_at_100_max
|
|
value: 32.05464592020074
|
|
- type: nauc_precision_at_100_std
|
|
value: 60.832570595613554
|
|
- type: nauc_precision_at_10_diff1
|
|
value: -11.91731376599939
|
|
- type: nauc_precision_at_10_max
|
|
value: 45.43646553157129
|
|
- type: nauc_precision_at_10_std
|
|
value: 52.962408871791276
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 80.99806778887115
|
|
- type: nauc_precision_at_1_max
|
|
value: 59.17855969530095
|
|
- type: nauc_precision_at_1_std
|
|
value: -12.30545640831458
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -18.43293701721667
|
|
- type: nauc_precision_at_20_max
|
|
value: 39.53434874203934
|
|
- type: nauc_precision_at_20_std
|
|
value: 53.6291982468461
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 30.84789043003892
|
|
- type: nauc_precision_at_3_max
|
|
value: 55.660727758110376
|
|
- type: nauc_precision_at_3_std
|
|
value: 17.87243920840355
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 4.099395181445625
|
|
- type: nauc_precision_at_5_max
|
|
value: 50.346770968709386
|
|
- type: nauc_precision_at_5_std
|
|
value: 44.66722483255029
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: .nan
|
|
- type: nauc_recall_at_1000_max
|
|
value: .nan
|
|
- type: nauc_recall_at_1000_std
|
|
value: .nan
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 100.0
|
|
- type: nauc_recall_at_100_max
|
|
value: 72.2222222222207
|
|
- type: nauc_recall_at_100_std
|
|
value: 86.92810457516407
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 62.18887555022005
|
|
- type: nauc_recall_at_10_max
|
|
value: 75.14339068960916
|
|
- type: nauc_recall_at_10_std
|
|
value: -1.4912631719357108
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 81.03932970557837
|
|
- type: nauc_recall_at_1_max
|
|
value: 49.02073230264529
|
|
- type: nauc_recall_at_1_std
|
|
value: -22.977452975845512
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 59.27414444038499
|
|
- type: nauc_recall_at_20_max
|
|
value: 76.32241302318047
|
|
- type: nauc_recall_at_20_std
|
|
value: -0.8322169447488666
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 69.58783002593157
|
|
- type: nauc_recall_at_3_max
|
|
value: 55.89660919896563
|
|
- type: nauc_recall_at_3_std
|
|
value: -21.183005510917862
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 65.53660499878802
|
|
- type: nauc_recall_at_5_max
|
|
value: 58.218018535135805
|
|
- type: nauc_recall_at_5_std
|
|
value: -8.328952210032455
|
|
- type: ndcg_at_1
|
|
value: 66.333
|
|
- type: ndcg_at_10
|
|
value: 80.08999999999999
|
|
- type: ndcg_at_100
|
|
value: 81.24900000000001
|
|
- type: ndcg_at_1000
|
|
value: 81.28800000000001
|
|
- type: ndcg_at_20
|
|
value: 80.625
|
|
- type: ndcg_at_3
|
|
value: 74.98700000000001
|
|
- type: ndcg_at_5
|
|
value: 78.553
|
|
- type: precision_at_1
|
|
value: 66.333
|
|
- type: precision_at_10
|
|
value: 10.667
|
|
- type: precision_at_100
|
|
value: 1.127
|
|
- type: precision_at_1000
|
|
value: 0.11299999999999999
|
|
- type: precision_at_20
|
|
value: 5.45
|
|
- type: precision_at_3
|
|
value: 29.555999999999997
|
|
- type: precision_at_5
|
|
value: 20.133000000000003
|
|
- type: recall_at_1
|
|
value: 63.161
|
|
- type: recall_at_10
|
|
value: 94.167
|
|
- type: recall_at_100
|
|
value: 99.667
|
|
- type: recall_at_1000
|
|
value: 100.0
|
|
- type: recall_at_20
|
|
value: 96.167
|
|
- type: recall_at_3
|
|
value: 80.972
|
|
- type: recall_at_5
|
|
value: 89.90599999999999
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB SprintDuplicateQuestions
|
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46
|
|
split: test
|
|
type: mteb/sprintduplicatequestions-pairclassification
|
|
metrics:
|
|
- type: cosine_accuracy
|
|
value: 99.81881188118813
|
|
- type: cosine_accuracy_threshold
|
|
value: 85.55081486701965
|
|
- type: cosine_ap
|
|
value: 96.0359661816236
|
|
- type: cosine_f1
|
|
value: 90.6584992343032
|
|
- type: cosine_f1_threshold
|
|
value: 84.82859134674072
|
|
- type: cosine_precision
|
|
value: 92.59645464025026
|
|
- type: cosine_recall
|
|
value: 88.8
|
|
- type: dot_accuracy
|
|
value: 99.81881188118813
|
|
- type: dot_accuracy_threshold
|
|
value: 84.91908311843872
|
|
- type: dot_ap
|
|
value: 96.05740121094365
|
|
- type: dot_f1
|
|
value: 90.81885856079404
|
|
- type: dot_f1_threshold
|
|
value: 83.84919166564941
|
|
- type: dot_precision
|
|
value: 90.14778325123153
|
|
- type: dot_recall
|
|
value: 91.5
|
|
- type: euclidean_accuracy
|
|
value: 99.82079207920792
|
|
- type: euclidean_accuracy_threshold
|
|
value: 54.49706315994263
|
|
- type: euclidean_ap
|
|
value: 96.03223527068818
|
|
- type: euclidean_f1
|
|
value: 90.72270630445925
|
|
- type: euclidean_f1_threshold
|
|
value: 54.49706315994263
|
|
- type: euclidean_precision
|
|
value: 93.05993690851734
|
|
- type: euclidean_recall
|
|
value: 88.5
|
|
- type: main_score
|
|
value: 96.32671902439806
|
|
- type: manhattan_accuracy
|
|
value: 99.83267326732673
|
|
- type: manhattan_accuracy_threshold
|
|
value: 3818.192672729492
|
|
- type: manhattan_ap
|
|
value: 96.32671902439806
|
|
- type: manhattan_f1
|
|
value: 91.52032112393378
|
|
- type: manhattan_f1_threshold
|
|
value: 3818.192672729492
|
|
- type: manhattan_precision
|
|
value: 91.8429003021148
|
|
- type: manhattan_recall
|
|
value: 91.2
|
|
- type: max_ap
|
|
value: 96.32671902439806
|
|
- type: max_f1
|
|
value: 91.52032112393378
|
|
- type: max_precision
|
|
value: 93.05993690851734
|
|
- type: max_recall
|
|
value: 91.5
|
|
- type: similarity_accuracy
|
|
value: 99.81881188118813
|
|
- type: similarity_accuracy_threshold
|
|
value: 85.55081486701965
|
|
- type: similarity_ap
|
|
value: 96.0359661816236
|
|
- type: similarity_f1
|
|
value: 90.6584992343032
|
|
- type: similarity_f1_threshold
|
|
value: 84.82859134674072
|
|
- type: similarity_precision
|
|
value: 92.59645464025026
|
|
- type: similarity_recall
|
|
value: 88.8
|
|
task:
|
|
type: PairClassification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB StackExchangeClustering
|
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259
|
|
split: test
|
|
type: mteb/stackexchange-clustering
|
|
metrics:
|
|
- type: main_score
|
|
value: 80.28558559137414
|
|
- type: v_measure
|
|
value: 80.28558559137414
|
|
- type: v_measure_std
|
|
value: 2.795276520287584
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB StackExchangeClusteringP2P
|
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44
|
|
split: test
|
|
type: mteb/stackexchange-clustering-p2p
|
|
metrics:
|
|
- type: main_score
|
|
value: 49.57135582416209
|
|
- type: v_measure
|
|
value: 49.57135582416209
|
|
- type: v_measure_std
|
|
value: 1.6414135468423754
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB StackOverflowDupQuestions
|
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69
|
|
split: test
|
|
type: mteb/stackoverflowdupquestions-reranking
|
|
metrics:
|
|
- type: main_score
|
|
value: 55.253002583598644
|
|
- type: map
|
|
value: 55.253002583598644
|
|
- type: mrr
|
|
value: 56.24172396231219
|
|
- type: nAUC_map_diff1
|
|
value: 40.00053248203427
|
|
- type: nAUC_map_max
|
|
value: 10.05441740585869
|
|
- type: nAUC_map_std
|
|
value: 8.227169286387552
|
|
- type: nAUC_mrr_diff1
|
|
value: 40.250446264233744
|
|
- type: nAUC_mrr_max
|
|
value: 10.586310195339053
|
|
- type: nAUC_mrr_std
|
|
value: 8.47326494370076
|
|
task:
|
|
type: Reranking
|
|
- dataset:
|
|
config: default
|
|
name: MTEB SummEval
|
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c
|
|
split: test
|
|
type: mteb/summeval
|
|
metrics:
|
|
- type: cosine_pearson
|
|
value: 31.19874648747059
|
|
- type: cosine_spearman
|
|
value: 31.493550648844863
|
|
- type: dot_pearson
|
|
value: 31.157847680289407
|
|
- type: dot_spearman
|
|
value: 31.575299712180538
|
|
- type: main_score
|
|
value: 31.493550648844863
|
|
- type: pearson
|
|
value: 31.19874648747059
|
|
- type: spearman
|
|
value: 31.493550648844863
|
|
task:
|
|
type: Summarization
|
|
- dataset:
|
|
config: default
|
|
name: MTEB TRECCOVID
|
|
revision: bb9466bac8153a0349341eb1b22e06409e78ef4e
|
|
split: test
|
|
type: mteb/trec-covid
|
|
metrics:
|
|
- type: main_score
|
|
value: 85.983
|
|
- type: map_at_1
|
|
value: 0.247
|
|
- type: map_at_10
|
|
value: 2.177
|
|
- type: map_at_100
|
|
value: 14.804
|
|
- type: map_at_1000
|
|
value: 37.045
|
|
- type: map_at_20
|
|
value: 4.12
|
|
- type: map_at_3
|
|
value: 0.7000000000000001
|
|
- type: map_at_5
|
|
value: 1.1320000000000001
|
|
- type: mrr_at_1
|
|
value: 96.0
|
|
- type: mrr_at_10
|
|
value: 98.0
|
|
- type: mrr_at_100
|
|
value: 98.0
|
|
- type: mrr_at_1000
|
|
value: 98.0
|
|
- type: mrr_at_20
|
|
value: 98.0
|
|
- type: mrr_at_3
|
|
value: 98.0
|
|
- type: mrr_at_5
|
|
value: 98.0
|
|
- type: nauc_map_at_1000_diff1
|
|
value: -0.9165125200337213
|
|
- type: nauc_map_at_1000_max
|
|
value: 40.260117798042764
|
|
- type: nauc_map_at_1000_std
|
|
value: 71.72789335831554
|
|
- type: nauc_map_at_100_diff1
|
|
value: 20.493827311583953
|
|
- type: nauc_map_at_100_max
|
|
value: 21.005742079276462
|
|
- type: nauc_map_at_100_std
|
|
value: 62.53815607831659
|
|
- type: nauc_map_at_10_diff1
|
|
value: 31.289297684528215
|
|
- type: nauc_map_at_10_max
|
|
value: 7.86554294370268
|
|
- type: nauc_map_at_10_std
|
|
value: 37.26191657133897
|
|
- type: nauc_map_at_1_diff1
|
|
value: 25.57568148849456
|
|
- type: nauc_map_at_1_max
|
|
value: -5.9767435623941445
|
|
- type: nauc_map_at_1_std
|
|
value: 30.849871717506755
|
|
- type: nauc_map_at_20_diff1
|
|
value: 30.896018204532087
|
|
- type: nauc_map_at_20_max
|
|
value: 8.667077299744314
|
|
- type: nauc_map_at_20_std
|
|
value: 41.512687168412924
|
|
- type: nauc_map_at_3_diff1
|
|
value: 29.44724521006598
|
|
- type: nauc_map_at_3_max
|
|
value: 1.597496889532064
|
|
- type: nauc_map_at_3_std
|
|
value: 32.25013773854697
|
|
- type: nauc_map_at_5_diff1
|
|
value: 27.387036605618825
|
|
- type: nauc_map_at_5_max
|
|
value: 5.402983746211454
|
|
- type: nauc_map_at_5_std
|
|
value: 33.940523962472184
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: -14.122315592903503
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 33.84687208216605
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 86.11111111111092
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: -14.122315592903503
|
|
- type: nauc_mrr_at_100_max
|
|
value: 33.84687208216605
|
|
- type: nauc_mrr_at_100_std
|
|
value: 86.11111111111092
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: -14.122315592903503
|
|
- type: nauc_mrr_at_10_max
|
|
value: 33.84687208216605
|
|
- type: nauc_mrr_at_10_std
|
|
value: 86.11111111111092
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: -14.122315592903831
|
|
- type: nauc_mrr_at_1_max
|
|
value: 33.84687208216637
|
|
- type: nauc_mrr_at_1_std
|
|
value: 86.11111111111124
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: -14.122315592903503
|
|
- type: nauc_mrr_at_20_max
|
|
value: 33.84687208216605
|
|
- type: nauc_mrr_at_20_std
|
|
value: 86.11111111111092
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: -14.122315592903503
|
|
- type: nauc_mrr_at_3_max
|
|
value: 33.84687208216605
|
|
- type: nauc_mrr_at_3_std
|
|
value: 86.11111111111092
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: -14.122315592903503
|
|
- type: nauc_mrr_at_5_max
|
|
value: 33.84687208216605
|
|
- type: nauc_mrr_at_5_std
|
|
value: 86.11111111111092
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 8.745907669561928
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 45.43307237994533
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 74.93357447176336
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: -3.9719350773353765
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 44.43705332397461
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 61.59493812371758
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 15.230915878367348
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 48.332840970836635
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 46.888785065125774
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 13.219732337379442
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 45.19919078742603
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 64.68253968253977
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 12.479648691964865
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 48.76688248450331
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 51.450399755887545
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 6.165414201871464
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 45.089689347691035
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 41.08249161845213
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 7.411245806844721
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 47.818748093538076
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 45.907685763676575
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -30.574290219847345
|
|
- type: nauc_precision_at_1000_max
|
|
value: 32.56926126118719
|
|
- type: nauc_precision_at_1000_std
|
|
value: 14.584504392628874
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -10.199740234718847
|
|
- type: nauc_precision_at_100_max
|
|
value: 41.0213226769777
|
|
- type: nauc_precision_at_100_std
|
|
value: 56.975760776771324
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 7.865792689701161
|
|
- type: nauc_precision_at_10_max
|
|
value: 52.00432275201737
|
|
- type: nauc_precision_at_10_std
|
|
value: 43.89512276413724
|
|
- type: nauc_precision_at_1_diff1
|
|
value: -14.122315592903831
|
|
- type: nauc_precision_at_1_max
|
|
value: 33.84687208216637
|
|
- type: nauc_precision_at_1_std
|
|
value: 86.11111111111124
|
|
- type: nauc_precision_at_20_diff1
|
|
value: 5.481424191880084
|
|
- type: nauc_precision_at_20_max
|
|
value: 46.86629331792725
|
|
- type: nauc_precision_at_20_std
|
|
value: 49.245692667517496
|
|
- type: nauc_precision_at_3_diff1
|
|
value: -5.870408807869163
|
|
- type: nauc_precision_at_3_max
|
|
value: 48.73657612128875
|
|
- type: nauc_precision_at_3_std
|
|
value: 41.15152062088262
|
|
- type: nauc_precision_at_5_diff1
|
|
value: -4.550610529125413
|
|
- type: nauc_precision_at_5_max
|
|
value: 60.390115878205386
|
|
- type: nauc_precision_at_5_std
|
|
value: 44.16494295055696
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: 8.047794367079034
|
|
- type: nauc_recall_at_1000_max
|
|
value: 37.07551482870489
|
|
- type: nauc_recall_at_1000_std
|
|
value: 66.20862163364201
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 25.08104923597475
|
|
- type: nauc_recall_at_100_max
|
|
value: 9.971294642165734
|
|
- type: nauc_recall_at_100_std
|
|
value: 51.737814074891254
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 32.33148478369628
|
|
- type: nauc_recall_at_10_max
|
|
value: 1.3767192150014917
|
|
- type: nauc_recall_at_10_std
|
|
value: 30.801926742876308
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 25.57568148849456
|
|
- type: nauc_recall_at_1_max
|
|
value: -5.9767435623941445
|
|
- type: nauc_recall_at_1_std
|
|
value: 30.849871717506755
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 31.716580022934654
|
|
- type: nauc_recall_at_20_max
|
|
value: -0.1281270579464631
|
|
- type: nauc_recall_at_20_std
|
|
value: 33.76185294993676
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 29.758810004388348
|
|
- type: nauc_recall_at_3_max
|
|
value: -1.9442985017191816
|
|
- type: nauc_recall_at_3_std
|
|
value: 27.45550076962206
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 27.047710181576672
|
|
- type: nauc_recall_at_5_max
|
|
value: 1.5237000700880248
|
|
- type: nauc_recall_at_5_std
|
|
value: 28.235297950159698
|
|
- type: ndcg_at_1
|
|
value: 94.0
|
|
- type: ndcg_at_10
|
|
value: 85.983
|
|
- type: ndcg_at_100
|
|
value: 69.195
|
|
- type: ndcg_at_1000
|
|
value: 62.541000000000004
|
|
- type: ndcg_at_20
|
|
value: 83.405
|
|
- type: ndcg_at_3
|
|
value: 89.98899999999999
|
|
- type: ndcg_at_5
|
|
value: 87.905
|
|
- type: precision_at_1
|
|
value: 96.0
|
|
- type: precision_at_10
|
|
value: 89.4
|
|
- type: precision_at_100
|
|
value: 71.54
|
|
- type: precision_at_1000
|
|
value: 27.594
|
|
- type: precision_at_20
|
|
value: 87.2
|
|
- type: precision_at_3
|
|
value: 92.667
|
|
- type: precision_at_5
|
|
value: 90.8
|
|
- type: recall_at_1
|
|
value: 0.247
|
|
- type: recall_at_10
|
|
value: 2.315
|
|
- type: recall_at_100
|
|
value: 17.574
|
|
- type: recall_at_1000
|
|
value: 59.336999999999996
|
|
- type: recall_at_20
|
|
value: 4.491
|
|
- type: recall_at_3
|
|
value: 0.7250000000000001
|
|
- type: recall_at_5
|
|
value: 1.1820000000000002
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB Touche2020
|
|
revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f
|
|
split: test
|
|
type: mteb/touche2020
|
|
metrics:
|
|
- type: main_score
|
|
value: 29.944
|
|
- type: map_at_1
|
|
value: 3.064
|
|
- type: map_at_10
|
|
value: 11.501999999999999
|
|
- type: map_at_100
|
|
value: 18.736
|
|
- type: map_at_1000
|
|
value: 20.333000000000002
|
|
- type: map_at_20
|
|
value: 14.057
|
|
- type: map_at_3
|
|
value: 6.300999999999999
|
|
- type: map_at_5
|
|
value: 8.463
|
|
- type: mrr_at_1
|
|
value: 44.89795918367347
|
|
- type: mrr_at_10
|
|
value: 58.41188856494979
|
|
- type: mrr_at_100
|
|
value: 58.93964266413245
|
|
- type: mrr_at_1000
|
|
value: 58.93964266413245
|
|
- type: mrr_at_20
|
|
value: 58.767485349118
|
|
- type: mrr_at_3
|
|
value: 54.42176870748299
|
|
- type: mrr_at_5
|
|
value: 56.666666666666664
|
|
- type: nauc_map_at_1000_diff1
|
|
value: 11.478593385608479
|
|
- type: nauc_map_at_1000_max
|
|
value: 10.309889845044324
|
|
- type: nauc_map_at_1000_std
|
|
value: 21.16721939940238
|
|
- type: nauc_map_at_100_diff1
|
|
value: 11.570438543562418
|
|
- type: nauc_map_at_100_max
|
|
value: 8.426183648064834
|
|
- type: nauc_map_at_100_std
|
|
value: 18.56231985033613
|
|
- type: nauc_map_at_10_diff1
|
|
value: 22.37735506247481
|
|
- type: nauc_map_at_10_max
|
|
value: 5.455946239060806
|
|
- type: nauc_map_at_10_std
|
|
value: -4.2848826518388154
|
|
- type: nauc_map_at_1_diff1
|
|
value: 27.853645380676824
|
|
- type: nauc_map_at_1_max
|
|
value: 7.30739948053113
|
|
- type: nauc_map_at_1_std
|
|
value: -0.2773663157814586
|
|
- type: nauc_map_at_20_diff1
|
|
value: 14.724669779924648
|
|
- type: nauc_map_at_20_max
|
|
value: 10.12882779173533
|
|
- type: nauc_map_at_20_std
|
|
value: 4.4803777672120875
|
|
- type: nauc_map_at_3_diff1
|
|
value: 31.891173385921263
|
|
- type: nauc_map_at_3_max
|
|
value: 4.889652271827218
|
|
- type: nauc_map_at_3_std
|
|
value: -9.477460238651643
|
|
- type: nauc_map_at_5_diff1
|
|
value: 31.489012040465003
|
|
- type: nauc_map_at_5_max
|
|
value: 1.7330092417337482
|
|
- type: nauc_map_at_5_std
|
|
value: -8.137018608469637
|
|
- type: nauc_mrr_at_1000_diff1
|
|
value: 24.411522237082416
|
|
- type: nauc_mrr_at_1000_max
|
|
value: 11.286971076556688
|
|
- type: nauc_mrr_at_1000_std
|
|
value: 23.443174210894043
|
|
- type: nauc_mrr_at_100_diff1
|
|
value: 24.411522237082416
|
|
- type: nauc_mrr_at_100_max
|
|
value: 11.286971076556688
|
|
- type: nauc_mrr_at_100_std
|
|
value: 23.443174210894043
|
|
- type: nauc_mrr_at_10_diff1
|
|
value: 23.948152308265186
|
|
- type: nauc_mrr_at_10_max
|
|
value: 12.22420979621155
|
|
- type: nauc_mrr_at_10_std
|
|
value: 23.557939024705544
|
|
- type: nauc_mrr_at_1_diff1
|
|
value: 17.902334894536107
|
|
- type: nauc_mrr_at_1_max
|
|
value: 17.36969662861018
|
|
- type: nauc_mrr_at_1_std
|
|
value: 19.425714969048734
|
|
- type: nauc_mrr_at_20_diff1
|
|
value: 24.635893795899797
|
|
- type: nauc_mrr_at_20_max
|
|
value: 11.330541067194913
|
|
- type: nauc_mrr_at_20_std
|
|
value: 23.74518583400233
|
|
- type: nauc_mrr_at_3_diff1
|
|
value: 25.045536328282587
|
|
- type: nauc_mrr_at_3_max
|
|
value: 7.497967004732733
|
|
- type: nauc_mrr_at_3_std
|
|
value: 24.167153007320078
|
|
- type: nauc_mrr_at_5_diff1
|
|
value: 24.328479930592454
|
|
- type: nauc_mrr_at_5_max
|
|
value: 10.037126854938336
|
|
- type: nauc_mrr_at_5_std
|
|
value: 25.236208055346136
|
|
- type: nauc_ndcg_at_1000_diff1
|
|
value: 15.555347444667389
|
|
- type: nauc_ndcg_at_1000_max
|
|
value: 13.356591700655718
|
|
- type: nauc_ndcg_at_1000_std
|
|
value: 42.42395845935052
|
|
- type: nauc_ndcg_at_100_diff1
|
|
value: 13.110526060413708
|
|
- type: nauc_ndcg_at_100_max
|
|
value: 3.140006440162515
|
|
- type: nauc_ndcg_at_100_std
|
|
value: 39.02733288398033
|
|
- type: nauc_ndcg_at_10_diff1
|
|
value: 20.68853369009725
|
|
- type: nauc_ndcg_at_10_max
|
|
value: 2.435389817058852
|
|
- type: nauc_ndcg_at_10_std
|
|
value: 10.038202768784316
|
|
- type: nauc_ndcg_at_1_diff1
|
|
value: 20.17287594582385
|
|
- type: nauc_ndcg_at_1_max
|
|
value: 12.487205168273196
|
|
- type: nauc_ndcg_at_1_std
|
|
value: 20.639827614373075
|
|
- type: nauc_ndcg_at_20_diff1
|
|
value: 16.987577348502985
|
|
- type: nauc_ndcg_at_20_max
|
|
value: 2.9978717644469266
|
|
- type: nauc_ndcg_at_20_std
|
|
value: 13.015690866750354
|
|
- type: nauc_ndcg_at_3_diff1
|
|
value: 32.392223079245575
|
|
- type: nauc_ndcg_at_3_max
|
|
value: 1.587587110582544
|
|
- type: nauc_ndcg_at_3_std
|
|
value: 12.850592473446609
|
|
- type: nauc_ndcg_at_5_diff1
|
|
value: 32.80244517369626
|
|
- type: nauc_ndcg_at_5_max
|
|
value: 5.8939933777508084
|
|
- type: nauc_ndcg_at_5_std
|
|
value: 15.779687411463414
|
|
- type: nauc_precision_at_1000_diff1
|
|
value: -14.314031720452537
|
|
- type: nauc_precision_at_1000_max
|
|
value: 32.87886666567266
|
|
- type: nauc_precision_at_1000_std
|
|
value: 21.49347046886851
|
|
- type: nauc_precision_at_100_diff1
|
|
value: -9.4034008613839
|
|
- type: nauc_precision_at_100_max
|
|
value: 16.784075123309645
|
|
- type: nauc_precision_at_100_std
|
|
value: 73.14688535393604
|
|
- type: nauc_precision_at_10_diff1
|
|
value: 6.855101404043058
|
|
- type: nauc_precision_at_10_max
|
|
value: 6.52491228645612
|
|
- type: nauc_precision_at_10_std
|
|
value: 16.104602266016744
|
|
- type: nauc_precision_at_1_diff1
|
|
value: 17.902334894536107
|
|
- type: nauc_precision_at_1_max
|
|
value: 17.36969662861018
|
|
- type: nauc_precision_at_1_std
|
|
value: 19.425714969048734
|
|
- type: nauc_precision_at_20_diff1
|
|
value: -5.337534613602212
|
|
- type: nauc_precision_at_20_max
|
|
value: 17.722925454767218
|
|
- type: nauc_precision_at_20_std
|
|
value: 34.26680462132849
|
|
- type: nauc_precision_at_3_diff1
|
|
value: 31.054623397809255
|
|
- type: nauc_precision_at_3_max
|
|
value: -0.92038600946826
|
|
- type: nauc_precision_at_3_std
|
|
value: 8.326997076862916
|
|
- type: nauc_precision_at_5_diff1
|
|
value: 29.784942296920462
|
|
- type: nauc_precision_at_5_max
|
|
value: 6.337469263434779
|
|
- type: nauc_precision_at_5_std
|
|
value: 12.789597196020974
|
|
- type: nauc_recall_at_1000_diff1
|
|
value: -3.8177981862041364
|
|
- type: nauc_recall_at_1000_max
|
|
value: 14.206064332229163
|
|
- type: nauc_recall_at_1000_std
|
|
value: 74.18853420771269
|
|
- type: nauc_recall_at_100_diff1
|
|
value: 0.7677996771461106
|
|
- type: nauc_recall_at_100_max
|
|
value: -4.139924106878441
|
|
- type: nauc_recall_at_100_std
|
|
value: 48.319930706362896
|
|
- type: nauc_recall_at_10_diff1
|
|
value: 12.038835537494322
|
|
- type: nauc_recall_at_10_max
|
|
value: -2.0498983557854418
|
|
- type: nauc_recall_at_10_std
|
|
value: -2.0339180690854493
|
|
- type: nauc_recall_at_1_diff1
|
|
value: 27.853645380676824
|
|
- type: nauc_recall_at_1_max
|
|
value: 7.30739948053113
|
|
- type: nauc_recall_at_1_std
|
|
value: -0.2773663157814586
|
|
- type: nauc_recall_at_20_diff1
|
|
value: 0.7907893667756708
|
|
- type: nauc_recall_at_20_max
|
|
value: 0.8795499810558195
|
|
- type: nauc_recall_at_20_std
|
|
value: 11.512483291688282
|
|
- type: nauc_recall_at_3_diff1
|
|
value: 33.19440392639576
|
|
- type: nauc_recall_at_3_max
|
|
value: -1.5494237697432613
|
|
- type: nauc_recall_at_3_std
|
|
value: -8.560408808376984
|
|
- type: nauc_recall_at_5_diff1
|
|
value: 27.42193873870941
|
|
- type: nauc_recall_at_5_max
|
|
value: -4.74350293281128
|
|
- type: nauc_recall_at_5_std
|
|
value: -7.618060131179654
|
|
- type: ndcg_at_1
|
|
value: 42.857
|
|
- type: ndcg_at_10
|
|
value: 29.944
|
|
- type: ndcg_at_100
|
|
value: 42.624
|
|
- type: ndcg_at_1000
|
|
value: 53.384
|
|
- type: ndcg_at_20
|
|
value: 30.135
|
|
- type: ndcg_at_3
|
|
value: 34.847
|
|
- type: ndcg_at_5
|
|
value: 32.573
|
|
- type: precision_at_1
|
|
value: 44.897999999999996
|
|
- type: precision_at_10
|
|
value: 25.306
|
|
- type: precision_at_100
|
|
value: 8.694
|
|
- type: precision_at_1000
|
|
value: 1.616
|
|
- type: precision_at_20
|
|
value: 19.082
|
|
- type: precision_at_3
|
|
value: 34.014
|
|
- type: precision_at_5
|
|
value: 31.019999999999996
|
|
- type: recall_at_1
|
|
value: 3.064
|
|
- type: recall_at_10
|
|
value: 17.849999999999998
|
|
- type: recall_at_100
|
|
value: 53.217999999999996
|
|
- type: recall_at_1000
|
|
value: 87.095
|
|
- type: recall_at_20
|
|
value: 26.111
|
|
- type: recall_at_3
|
|
value: 7.383000000000001
|
|
- type: recall_at_5
|
|
value: 11.434
|
|
task:
|
|
type: Retrieval
|
|
- dataset:
|
|
config: default
|
|
name: MTEB ToxicConversationsClassification
|
|
revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de
|
|
split: test
|
|
type: mteb/toxic_conversations_50k
|
|
metrics:
|
|
- type: accuracy
|
|
value: 88.759765625
|
|
- type: ap
|
|
value: 36.49152357863017
|
|
- type: ap_weighted
|
|
value: 36.49152357863017
|
|
- type: f1
|
|
value: 74.4692714448641
|
|
- type: f1_weighted
|
|
value: 90.54372649306606
|
|
- type: main_score
|
|
value: 88.759765625
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB TweetSentimentExtractionClassification
|
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a
|
|
split: test
|
|
type: mteb/tweet_sentiment_extraction
|
|
metrics:
|
|
- type: accuracy
|
|
value: 74.8443689869836
|
|
- type: f1
|
|
value: 75.1139662898148
|
|
- type: f1_weighted
|
|
value: 74.7369003946243
|
|
- type: main_score
|
|
value: 74.8443689869836
|
|
task:
|
|
type: Classification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB TwentyNewsgroupsClustering
|
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449
|
|
split: test
|
|
type: mteb/twentynewsgroups-clustering
|
|
metrics:
|
|
- type: main_score
|
|
value: 61.42918790942448
|
|
- type: v_measure
|
|
value: 61.42918790942448
|
|
- type: v_measure_std
|
|
value: 1.0156550098843082
|
|
task:
|
|
type: Clustering
|
|
- dataset:
|
|
config: default
|
|
name: MTEB TwitterSemEval2015
|
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1
|
|
split: test
|
|
type: mteb/twittersemeval2015-pairclassification
|
|
metrics:
|
|
- type: cosine_accuracy
|
|
value: 88.22197055492639
|
|
- type: cosine_accuracy_threshold
|
|
value: 83.30042362213135
|
|
- type: cosine_ap
|
|
value: 80.57754959194938
|
|
- type: cosine_f1
|
|
value: 73.70579190158894
|
|
- type: cosine_f1_threshold
|
|
value: 81.04978799819946
|
|
- type: cosine_precision
|
|
value: 71.64922770303936
|
|
- type: cosine_recall
|
|
value: 75.8839050131926
|
|
- type: dot_accuracy
|
|
value: 88.23985217857782
|
|
- type: dot_accuracy_threshold
|
|
value: 83.31039547920227
|
|
- type: dot_ap
|
|
value: 80.57533213448181
|
|
- type: dot_f1
|
|
value: 73.61309601143302
|
|
- type: dot_f1_threshold
|
|
value: 81.33968114852905
|
|
- type: dot_precision
|
|
value: 72.51087791144101
|
|
- type: dot_recall
|
|
value: 74.74934036939314
|
|
- type: euclidean_accuracy
|
|
value: 88.22197055492639
|
|
- type: euclidean_accuracy_threshold
|
|
value: 58.290231227874756
|
|
- type: euclidean_ap
|
|
value: 80.57982723880139
|
|
- type: euclidean_f1
|
|
value: 73.63426519620417
|
|
- type: euclidean_f1_threshold
|
|
value: 61.55576705932617
|
|
- type: euclidean_precision
|
|
value: 71.63173652694611
|
|
- type: euclidean_recall
|
|
value: 75.75197889182058
|
|
- type: main_score
|
|
value: 80.57982723880139
|
|
- type: manhattan_accuracy
|
|
value: 88.14448351910353
|
|
- type: manhattan_accuracy_threshold
|
|
value: 3907.2471618652344
|
|
- type: manhattan_ap
|
|
value: 80.3538079655539
|
|
- type: manhattan_f1
|
|
value: 73.40466675261054
|
|
- type: manhattan_f1_threshold
|
|
value: 4103.794097900391
|
|
- type: manhattan_precision
|
|
value: 71.76707839677337
|
|
- type: manhattan_recall
|
|
value: 75.11873350923483
|
|
- type: max_ap
|
|
value: 80.57982723880139
|
|
- type: max_f1
|
|
value: 73.70579190158894
|
|
- type: max_precision
|
|
value: 72.51087791144101
|
|
- type: max_recall
|
|
value: 75.8839050131926
|
|
- type: similarity_accuracy
|
|
value: 88.22197055492639
|
|
- type: similarity_accuracy_threshold
|
|
value: 83.30042362213135
|
|
- type: similarity_ap
|
|
value: 80.57754959194938
|
|
- type: similarity_f1
|
|
value: 73.70579190158894
|
|
- type: similarity_f1_threshold
|
|
value: 81.04978799819946
|
|
- type: similarity_precision
|
|
value: 71.64922770303936
|
|
- type: similarity_recall
|
|
value: 75.8839050131926
|
|
task:
|
|
type: PairClassification
|
|
- dataset:
|
|
config: default
|
|
name: MTEB TwitterURLCorpus
|
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf
|
|
split: test
|
|
type: mteb/twitterurlcorpus-pairclassification
|
|
metrics:
|
|
- type: cosine_accuracy
|
|
value: 89.88628866379477
|
|
- type: cosine_accuracy_threshold
|
|
value: 80.8050274848938
|
|
- type: cosine_ap
|
|
value: 87.57594591596816
|
|
- type: cosine_f1
|
|
value: 80.0812257707218
|
|
- type: cosine_f1_threshold
|
|
value: 77.990061044693
|
|
- type: cosine_precision
|
|
value: 76.93126197063205
|
|
- type: cosine_recall
|
|
value: 83.50015398829689
|
|
- type: dot_accuracy
|
|
value: 89.87852679784221
|
|
- type: dot_accuracy_threshold
|
|
value: 80.84419965744019
|
|
- type: dot_ap
|
|
value: 87.56136742222151
|
|
- type: dot_f1
|
|
value: 80.05898617511521
|
|
- type: dot_f1_threshold
|
|
value: 77.92385816574097
|
|
- type: dot_precision
|
|
value: 76.80554573106035
|
|
- type: dot_recall
|
|
value: 83.60024638127503
|
|
- type: euclidean_accuracy
|
|
value: 89.86882446540149
|
|
- type: euclidean_accuracy_threshold
|
|
value: 62.08193898200989
|
|
- type: euclidean_ap
|
|
value: 87.57517549192228
|
|
- type: euclidean_f1
|
|
value: 80.05286925872892
|
|
- type: euclidean_f1_threshold
|
|
value: 66.65036082267761
|
|
- type: euclidean_precision
|
|
value: 76.51063232507545
|
|
- type: euclidean_recall
|
|
value: 83.93902063443178
|
|
- type: main_score
|
|
value: 87.64162614197194
|
|
- type: manhattan_accuracy
|
|
value: 89.8959909962355
|
|
- type: manhattan_accuracy_threshold
|
|
value: 4176.108169555664
|
|
- type: manhattan_ap
|
|
value: 87.64162614197194
|
|
- type: manhattan_f1
|
|
value: 80.17116279069768
|
|
- type: manhattan_f1_threshold
|
|
value: 4433.153533935547
|
|
- type: manhattan_precision
|
|
value: 77.57615035644848
|
|
- type: manhattan_recall
|
|
value: 82.94579611949491
|
|
- type: max_ap
|
|
value: 87.64162614197194
|
|
- type: max_f1
|
|
value: 80.17116279069768
|
|
- type: max_precision
|
|
value: 77.57615035644848
|
|
- type: max_recall
|
|
value: 83.93902063443178
|
|
- type: similarity_accuracy
|
|
value: 89.88628866379477
|
|
- type: similarity_accuracy_threshold
|
|
value: 80.8050274848938
|
|
- type: similarity_ap
|
|
value: 87.57594591596816
|
|
- type: similarity_f1
|
|
value: 80.0812257707218
|
|
- type: similarity_f1_threshold
|
|
value: 77.990061044693
|
|
- type: similarity_precision
|
|
value: 76.93126197063205
|
|
- type: similarity_recall
|
|
value: 83.50015398829689
|
|
task:
|
|
type: PairClassification
|
|
tags:
|
|
- mteb
|
|
- sentence-transformers
|
|
- transformers
|
|
- sentence-similarity
|
|
license: mit
|
|
---
|
|
|
|
|
|
# Updates
|
|
|
|
New open-source models and ToDoList will be listed on https://github.com/DunZhang/Stella/blob/main/news_and_todo.md.
|
|
|
|
You can also find these models on my [homepage](https://huggingface.co/infgrad).
|
|
|
|
# Introduction
|
|
|
|
The models are trained based on `Alibaba-NLP/gte-large-en-v1.5` and `Alibaba-NLP/gte-Qwen2-1.5B-instruct`. Thanks for
|
|
their contributions!
|
|
|
|
**We simplify usage of prompts, providing two prompts for most general tasks, one is for s2p, another one is for s2s.**
|
|
|
|
Prompt of s2p task(e.g. retrieve task):
|
|
|
|
```text
|
|
Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: {query}
|
|
```
|
|
|
|
Prompt of s2s task(e.g. semantic textual similarity task):
|
|
|
|
```text
|
|
Instruct: Retrieve semantically similar text.\nQuery: {query}
|
|
```
|
|
|
|
The models are finally trained by [MRL]((https://arxiv.org/abs/2205.13147)), so they have multiple dimensions: 512, 768,
|
|
1024, 2048, 4096, 6144 and 8192.
|
|
|
|
The higher the dimension, the better the performance.
|
|
**Generally speaking, 1024d is good enough.** The MTEB score of 1024d is only 0.001 lower than 8192d.
|
|
|
|
# Model directory structure
|
|
|
|
The model directory structure is very simple, it is a standard SentenceTransformer directory **with a series
|
|
of `2_Dense_{dims}`
|
|
folders**, where `dims` represents the final vector dimension.
|
|
|
|
For example, the `2_Dense_256` folder stores Linear weights that convert vector dimensions to 256 dimensions.
|
|
Please refer to the following chapters for specific instructions on how to use them.
|
|
|
|
# Usage
|
|
|
|
You can use `SentenceTransformers` or `transformers` library to encode text.
|
|
|
|
## Sentence Transformers
|
|
|
|
```python
|
|
from sentence_transformers import SentenceTransformer
|
|
|
|
# This model supports two prompts: "s2p_query" and "s2s_query" for sentence-to-passage and sentence-to-sentence tasks, respectively.
|
|
# They are defined in `config_sentence_transformers.json`
|
|
query_prompt_name = "s2p_query"
|
|
queries = [
|
|
"What are some ways to reduce stress?",
|
|
"What are the benefits of drinking green tea?",
|
|
]
|
|
# docs do not need any prompts
|
|
docs = [
|
|
"There are many effective ways to reduce stress. Some common techniques include deep breathing, meditation, and physical activity. Engaging in hobbies, spending time in nature, and connecting with loved ones can also help alleviate stress. Additionally, setting boundaries, practicing self-care, and learning to say no can prevent stress from building up.",
|
|
"Green tea has been consumed for centuries and is known for its potential health benefits. It contains antioxidants that may help protect the body against damage caused by free radicals. Regular consumption of green tea has been associated with improved heart health, enhanced cognitive function, and a reduced risk of certain types of cancer. The polyphenols in green tea may also have anti-inflammatory and weight loss properties.",
|
|
]
|
|
|
|
# ！The default dimension is 1024, if you need other dimensions, please clone the model and modify `modules.json` to replace `2_Dense_1024` with another dimension, e.g. `2_Dense_256` or `2_Dense_8192` !
|
|
model = SentenceTransformer("infgrad/stella_en_1.5B_v5", trust_remote_code=True).cuda()
|
|
query_embeddings = model.encode(queries, prompt_name=query_prompt_name)
|
|
doc_embeddings = model.encode(docs)
|
|
print(query_embeddings.shape, doc_embeddings.shape)
|
|
# (2, 1024) (2, 1024)
|
|
|
|
similarities = model.similarity(query_embeddings, doc_embeddings)
|
|
print(similarities)
|
|
# tensor([[0.8179, 0.2958],
|
|
# [0.3194, 0.7854]])
|
|
```
|
|
|
|
## Transformers
|
|
|
|
```python
|
|
import os
|
|
import torch
|
|
from transformers import AutoModel, AutoTokenizer
|
|
from sklearn.preprocessing import normalize
|
|
|
|
query_prompt = "Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: "
|
|
queries = [
|
|
"What are some ways to reduce stress?",
|
|
"What are the benefits of drinking green tea?",
|
|
]
|
|
queries = [query_prompt + query for query in queries]
|
|
# docs do not need any prompts
|
|
docs = [
|
|
"There are many effective ways to reduce stress. Some common techniques include deep breathing, meditation, and physical activity. Engaging in hobbies, spending time in nature, and connecting with loved ones can also help alleviate stress. Additionally, setting boundaries, practicing self-care, and learning to say no can prevent stress from building up.",
|
|
"Green tea has been consumed for centuries and is known for its potential health benefits. It contains antioxidants that may help protect the body against damage caused by free radicals. Regular consumption of green tea has been associated with improved heart health, enhanced cognitive function, and a reduced risk of certain types of cancer. The polyphenols in green tea may also have anti-inflammatory and weight loss properties.",
|
|
]
|
|
|
|
# The path of your model after cloning it
|
|
model_dir = "{Your MODEL_PATH}"
|
|
|
|
vector_dim = 1024
|
|
vector_linear_directory = f"2_Dense_{vector_dim}"
|
|
model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).cuda().eval()
|
|
tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
|
|
vector_linear = torch.nn.Linear(in_features=model.config.hidden_size, out_features=vector_dim)
|
|
vector_linear_dict = {
|
|
k.replace("linear.", ""): v for k, v in
|
|
torch.load(os.path.join(model_dir, f"{vector_linear_directory}/pytorch_model.bin")).items()
|
|
}
|
|
vector_linear.load_state_dict(vector_linear_dict)
|
|
vector_linear.cuda()
|
|
|
|
# Embed the queries
|
|
with torch.no_grad():
|
|
input_data = tokenizer(queries, padding="longest", truncation=True, max_length=512, return_tensors="pt")
|
|
input_data = {k: v.cuda() for k, v in input_data.items()}
|
|
attention_mask = input_data["attention_mask"]
|
|
last_hidden_state = model(**input_data)[0]
|
|
last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
|
|
query_vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
|
|
query_vectors = normalize(vector_linear(query_vectors).cpu().numpy())
|
|
|
|
# Embed the documents
|
|
with torch.no_grad():
|
|
input_data = tokenizer(docs, padding="longest", truncation=True, max_length=512, return_tensors="pt")
|
|
input_data = {k: v.cuda() for k, v in input_data.items()}
|
|
attention_mask = input_data["attention_mask"]
|
|
last_hidden_state = model(**input_data)[0]
|
|
last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
|
|
docs_vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
|
|
docs_vectors = normalize(vector_linear(docs_vectors).cpu().numpy())
|
|
|
|
print(query_vectors.shape, docs_vectors.shape)
|
|
# (2, 1024) (2, 1024)
|
|
|
|
similarities = query_vectors @ docs_vectors.T
|
|
print(similarities)
|
|
# [[0.8178789 0.2958377 ]
|
|
# [0.31938642 0.7853526 ]]
|
|
```
|
|
|
|
# FAQ
|
|
|
|
Q: The details of training?
|
|
|
|
A: The training method and datasets will be released in the future. (specific time unknown, may be provided in a paper)
|
|
|
|
Q: How to choose a suitable prompt for my own task?
|
|
|
|
A: In most cases, please use the s2p and s2s prompts. These two prompts account for the vast majority of the training
|
|
data.
|
|
|
|
Q: How to reproduce MTEB results?
|
|
|
|
A: Please use evaluation scripts in `Alibaba-NLP/gte-Qwen2-1.5B-instruct` or `intfloat/e5-mistral-7b-instruct`
|
|
|
|
Q: Why each dimension has a linear weight?
|
|
|
|
A: MRL has multiple training methods, we choose this method which has the best performance.
|
|
|
|
Q: What is the sequence length of models?
|
|
|
|
A: 512 is recommended, in our experiments, almost all models perform poorly on specialized long text retrieval datasets. Besides, the
|
|
model is trained on datasets of 512 length. This may be an optimization term.
|
|
|
|
If you have any questions, please start a discussion on community. |
