results.json

{"tatdqa_test": {"ndcg_at_1": 0.55043, "ndcg_at_3": 0.65383, "ndcg_at_5": 0.67972, "ndcg_at_10": 0.70182, "ndcg_at_20": 0.71531, "ndcg_at_100": 0.72798, "ndcg_at_1000": 0.73147, "map_at_1": 0.55043, "map_at_3": 0.6292, "map_at_5": 0.64363, "map_at_10": 0.65292, "map_at_20": 0.6567, "map_at_100": 0.65856, "map_at_1000": 0.65873, "recall_at_1": 0.55043, "recall_at_3": 0.72479, "recall_at_5": 0.78736, "recall_at_10": 0.8548, "recall_at_20": 0.90765, "recall_at_100": 0.97448, "recall_at_1000": 1.0, "precision_at_1": 0.55043, "precision_at_3": 0.2416, "precision_at_5": 0.15747, "precision_at_10": 0.08548, "precision_at_20": 0.04538, "precision_at_100": 0.00974, "precision_at_1000": 0.001, "mrr_at_1": 0.5498177399756987, "mrr_at_3": 0.6287970838396112, "mrr_at_5": 0.6434690157958688, "mrr_at_10": 0.6530902235337229, "mrr_at_20": 0.6567284540447504, "mrr_at_100": 0.658621212229932, "mrr_at_1000": 0.6587698669596878, "naucs_at_1_max": 0.27853365504109656, "naucs_at_1_std": -0.15447634347947511, "naucs_at_1_diff1": 0.7134897043728883, "naucs_at_3_max": 0.2862020499335342, "naucs_at_3_std": -0.1406768618430069, "naucs_at_3_diff1": 0.598287834466956, "naucs_at_5_max": 0.3082772653205975, "naucs_at_5_std": -0.1336535664737251, "naucs_at_5_diff1": 0.5847681853576895, "naucs_at_10_max": 0.4156924004771727, "naucs_at_10_std": -0.012527556044571743, "naucs_at_10_diff1": 0.5762936692610346, "naucs_at_20_max": 0.39150222479435925, "naucs_at_20_std": 0.004434403605241619, "naucs_at_20_diff1": 0.5602018066874601, "naucs_at_100_max": 0.39176268171942447, "naucs_at_100_std": 0.360160411337065, "naucs_at_100_diff1": 0.6304563665269014, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shiftproject_test": {"ndcg_at_1": 0.63, "ndcg_at_3": 0.76488, "ndcg_at_5": 0.79072, "ndcg_at_10": 0.80085, "ndcg_at_20": 0.80877, "ndcg_at_100": 0.81273, "ndcg_at_1000": 0.81415, "map_at_1": 0.63, "map_at_3": 0.735, "map_at_5": 0.75, "map_at_10": 0.75444, "map_at_20": 0.75679, "map_at_100": 0.75742, "map_at_1000": 0.75749, "recall_at_1": 0.63, "recall_at_3": 0.85, "recall_at_5": 0.91, "recall_at_10": 0.94, "recall_at_20": 0.97, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.63, "precision_at_3": 0.28333, "precision_at_5": 0.182, "precision_at_10": 0.094, "precision_at_20": 0.0485, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.65, "mrr_at_3": 0.7483333333333333, "mrr_at_5": 0.7608333333333333, "mrr_at_10": 0.7663888888888889, "mrr_at_20": 0.7679646464646465, "mrr_at_100": 0.768591427091427, "mrr_at_1000": 0.7686683501683501, "naucs_at_1_max": -0.061987809273100325, "naucs_at_1_std": -0.18575972672715027, "naucs_at_1_diff1": 0.7242513984863435, "naucs_at_3_max": 0.18226488773185812, "naucs_at_3_std": -0.19902375528799124, "naucs_at_3_diff1": 0.6877969410999027, "naucs_at_5_max": -0.24074074074073906, "naucs_at_5_std": -0.7552131963896631, "naucs_at_5_diff1": 0.6890756302521003, "naucs_at_10_max": -0.3181605975723611, "naucs_at_10_std": -1.07002801120448, "naucs_at_10_diff1": 0.7005135387488336, "naucs_at_20_max": -0.7791783380018631, "naucs_at_20_std": -0.5961718020541553, "naucs_at_20_diff1": 0.401027077497665, "naucs_at_100_max": 0.554154995331464, "naucs_at_100_std": -0.17133520074697067, "naucs_at_100_diff1": 0.35807656395892007, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.98131, "ndcg_at_5": 0.98131, "ndcg_at_10": 0.98131, "ndcg_at_20": 0.98131, "ndcg_at_100": 0.98321, "ndcg_at_1000": 0.98321, "map_at_1": 0.97, "map_at_3": 0.97833, "map_at_5": 0.97833, "map_at_10": 0.97833, "map_at_20": 0.97833, "map_at_100": 0.9786, "map_at_1000": 0.9786, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.97, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_100": 0.9786111111111111, "mrr_at_1000": 0.9786111111111111, "naucs_at_1_max": 0.41518829754123937, "naucs_at_1_std": -1.1517273576097102, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 1.0, "naucs_at_3_std": -1.1517273576097802, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": -1.1517273576097316, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": -1.1517273576097316, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.94786, "ndcg_at_5": 0.94786, "ndcg_at_10": 0.95142, "ndcg_at_20": 0.95142, "ndcg_at_100": 0.95142, "ndcg_at_1000": 0.95142, "map_at_1": 0.89, "map_at_3": 0.93333, "map_at_5": 0.93333, "map_at_10": 0.935, "map_at_20": 0.935, "map_at_100": 0.935, "map_at_1000": 0.935, "recall_at_1": 0.89, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.89, "mrr_at_3": 0.935, "mrr_at_5": 0.937, "mrr_at_10": 0.937, "mrr_at_20": 0.937, "mrr_at_100": 0.937, "mrr_at_1000": 0.937, "naucs_at_1_max": 0.5626234859548148, "naucs_at_1_std": 0.3716175586289838, "naucs_at_1_diff1": 0.834850957821493, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "arxivqa_test_subsampled": {"ndcg_at_1": 0.702, "ndcg_at_3": 0.76743, "ndcg_at_5": 0.77991, "ndcg_at_10": 0.79468, "ndcg_at_20": 0.80462, "ndcg_at_100": 0.8145, "ndcg_at_1000": 0.81657, "map_at_1": 0.702, "map_at_3": 0.75133, "map_at_5": 0.75833, "map_at_10": 0.76436, "map_at_20": 0.767, "map_at_100": 0.76831, "map_at_1000": 0.76839, "recall_at_1": 0.702, "recall_at_3": 0.814, "recall_at_5": 0.844, "recall_at_10": 0.89, "recall_at_20": 0.93, "recall_at_100": 0.984, "recall_at_1000": 1.0, "precision_at_1": 0.702, "precision_at_3": 0.27133, "precision_at_5": 0.1688, "precision_at_10": 0.089, "precision_at_20": 0.0465, "precision_at_100": 0.00984, "precision_at_1000": 0.001, "mrr_at_1": 0.7, "mrr_at_3": 0.7506666666666666, "mrr_at_5": 0.7575666666666667, "mrr_at_10": 0.76395, "mrr_at_20": 0.7664683712486344, "mrr_at_100": 0.7677856040474413, "mrr_at_1000": 0.767866298103699, "naucs_at_1_max": 0.6661847878465089, "naucs_at_1_std": -0.217519374789405, "naucs_at_1_diff1": 0.9071749220117173, "naucs_at_3_max": 0.697658762809486, "naucs_at_3_std": -0.18659371894467894, "naucs_at_3_diff1": 0.839073335969141, "naucs_at_5_max": 0.7447876108417706, "naucs_at_5_std": -0.17636087316938498, "naucs_at_5_diff1": 0.8526087065738912, "naucs_at_10_max": 0.7873636285542479, "naucs_at_10_std": -0.15812215445408626, "naucs_at_10_diff1": 0.8351945709131511, "naucs_at_20_max": 0.7746565292783784, "naucs_at_20_std": -0.23473389355742383, "naucs_at_20_diff1": 0.8061224489795931, "naucs_at_100_max": 0.8027544351073763, "naucs_at_100_std": 0.2001050420168152, "naucs_at_100_diff1": 0.7639472455648932, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_test_subsampled": {"ndcg_at_1": 0.49446, "ndcg_at_3": 0.54968, "ndcg_at_5": 0.57172, "ndcg_at_10": 0.59184, "ndcg_at_20": 0.60528, "ndcg_at_100": 0.62838, "ndcg_at_1000": 0.64353, "map_at_1": 0.49446, "map_at_3": 0.53585, "map_at_5": 0.54815, "map_at_10": 0.55648, "map_at_20": 0.56018, "map_at_100": 0.56326, "map_at_1000": 0.56389, "recall_at_1": 0.49446, "recall_at_3": 0.5898, "recall_at_5": 0.64302, "recall_at_10": 0.7051, "recall_at_20": 0.75831, "recall_at_100": 0.8847, "recall_at_1000": 1.0, "precision_at_1": 0.49446, "precision_at_3": 0.1966, "precision_at_5": 0.1286, "precision_at_10": 0.07051, "precision_at_20": 0.03792, "precision_at_100": 0.00885, "precision_at_1000": 0.001, "mrr_at_1": 0.49223946784922396, "mrr_at_3": 0.5339985218033999, "mrr_at_5": 0.5473022912047303, "mrr_at_10": 0.5553206278816035, "mrr_at_20": 0.5589821565391849, "mrr_at_100": 0.5619819074681309, "mrr_at_1000": 0.5626456935597429, "naucs_at_1_max": 0.32878731695222985, "naucs_at_1_std": 0.15591990126747984, "naucs_at_1_diff1": 0.8493613636054403, "naucs_at_3_max": 0.23098165064202703, "naucs_at_3_std": 0.1729343384390047, "naucs_at_3_diff1": 0.79653911998206, "naucs_at_5_max": 0.20443193032581358, "naucs_at_5_std": 0.22886452804275878, "naucs_at_5_diff1": 0.7709831107458899, "naucs_at_10_max": 0.11430383031823631, "naucs_at_10_std": 0.3505331687518112, "naucs_at_10_diff1": 0.7655459781075918, "naucs_at_20_max": 0.06375294772557255, "naucs_at_20_std": 0.47875225635393476, "naucs_at_20_diff1": 0.7402047132383933, "naucs_at_100_max": -0.17203513994521832, "naucs_at_100_std": 0.719961371895444, "naucs_at_100_diff1": 0.7516431118242167, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.94, "ndcg_at_3": 0.96262, "ndcg_at_5": 0.96693, "ndcg_at_10": 0.96693, "ndcg_at_20": 0.96948, "ndcg_at_100": 0.96948, "ndcg_at_1000": 0.96948, "map_at_1": 0.94, "map_at_3": 0.95667, "map_at_5": 0.95917, "map_at_10": 0.95917, "map_at_20": 0.95988, "map_at_100": 0.95988, "map_at_1000": 0.95988, "recall_at_1": 0.94, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.96, "mrr_at_3": 0.9683333333333333, "mrr_at_5": 0.9708333333333333, "mrr_at_10": 0.9708333333333333, "mrr_at_20": 0.971547619047619, "mrr_at_100": 0.971547619047619, "mrr_at_1000": 0.971547619047619, "naucs_at_1_max": 0.4701991907874248, "naucs_at_1_std": -0.2544351073762836, "naucs_at_1_diff1": 0.9319172113289763, "naucs_at_3_max": -0.004668534080301393, "naucs_at_3_std": -0.5088702147525775, "naucs_at_3_diff1": 0.7957516339869218, "naucs_at_5_max": -0.5634920634920767, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 0.7222222222222276, "naucs_at_10_max": -0.5634920634920767, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 0.7222222222222276, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_test_subsampled": {"ndcg_at_1": 0.76518, "ndcg_at_3": 0.81846, "ndcg_at_5": 0.82779, "ndcg_at_10": 0.84084, "ndcg_at_20": 0.8454, "ndcg_at_100": 0.85306, "ndcg_at_1000": 0.85616, "map_at_1": 0.76518, "map_at_3": 0.80601, "map_at_5": 0.81127, "map_at_10": 0.81663, "map_at_20": 0.81786, "map_at_100": 0.81898, "map_at_1000": 0.81909, "recall_at_1": 0.76518, "recall_at_3": 0.85425, "recall_at_5": 0.87652, "recall_at_10": 0.917, "recall_at_20": 0.93522, "recall_at_100": 0.97571, "recall_at_1000": 1.0, "precision_at_1": 0.76518, "precision_at_3": 0.28475, "precision_at_5": 0.1753, "precision_at_10": 0.0917, "precision_at_20": 0.04676, "precision_at_100": 0.00976, "precision_at_1000": 0.001, "mrr_at_1": 0.7672064777327935, "mrr_at_3": 0.8076923076923077, "mrr_at_5": 0.8130566801619433, "mrr_at_10": 0.8183712486344065, "mrr_at_20": 0.8196021674007504, "mrr_at_100": 0.8207219702130965, "mrr_at_1000": 0.8208340179158132, "naucs_at_1_max": 0.526736932815251, "naucs_at_1_std": -0.08005662601905021, "naucs_at_1_diff1": 0.8906383040101385, "naucs_at_3_max": 0.5628217589502187, "naucs_at_3_std": 0.07419124093710198, "naucs_at_3_diff1": 0.819273316147957, "naucs_at_5_max": 0.6142211700114351, "naucs_at_5_std": 0.13484006376865668, "naucs_at_5_diff1": 0.7974348993177082, "naucs_at_10_max": 0.5107732925204674, "naucs_at_10_std": 0.08282924765220158, "naucs_at_10_diff1": 0.7501472816409511, "naucs_at_20_max": 0.6069714717598735, "naucs_at_20_std": 0.24818260935350467, "naucs_at_20_diff1": 0.7972653744711068, "naucs_at_100_max": 0.7715054503371943, "naucs_at_100_std": 0.6498035527508992, "naucs_at_100_diff1": 0.7627810617053321, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95155, "ndcg_at_5": 0.95155, "ndcg_at_10": 0.95155, "ndcg_at_20": 0.95425, "ndcg_at_100": 0.95618, "ndcg_at_1000": 0.95745, "map_at_1": 0.92, "map_at_3": 0.945, "map_at_5": 0.945, "map_at_10": 0.945, "map_at_20": 0.94583, "map_at_100": 0.94612, "map_at_1000": 0.94616, "recall_at_1": 0.92, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.92, "mrr_at_3": 0.945, "mrr_at_5": 0.945, "mrr_at_10": 0.945, "mrr_at_20": 0.9458333333333333, "mrr_at_100": 0.9461190476190476, "mrr_at_1000": 0.9461614205004035, "naucs_at_1_max": 0.5274859943977597, "naucs_at_1_std": -0.6526027077497683, "naucs_at_1_diff1": 0.9673202614379083, "naucs_at_3_max": 0.807812013694371, "naucs_at_3_std": -1.1517273576097098, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.807812013694364, "naucs_at_5_std": -1.1517273576097071, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.807812013694364, "naucs_at_10_std": -1.1517273576097071, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": -0.8576097105508901, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": -0.5634920634920583, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.85, "ndcg_at_3": 0.88465, "ndcg_at_5": 0.89664, "ndcg_at_10": 0.90491, "ndcg_at_20": 0.91464, "ndcg_at_100": 0.91464, "ndcg_at_1000": 0.91464, "map_at_1": 0.85, "map_at_3": 0.87679, "map_at_5": 0.88357, "map_at_10": 0.8871, "map_at_20": 0.88966, "map_at_100": 0.88966, "map_at_1000": 0.88966, "recall_at_1": 0.85, "recall_at_3": 0.90714, "recall_at_5": 0.93571, "recall_at_10": 0.96071, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.30238, "precision_at_5": 0.18714, "precision_at_10": 0.09607, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.85, "mrr_at_3": 0.8779761904761905, "mrr_at_5": 0.8838690476190476, "mrr_at_10": 0.8874900793650794, "mrr_at_20": 0.8898663632119514, "mrr_at_100": 0.8900364312391623, "mrr_at_1000": 0.8900364312391623, "naucs_at_1_max": 0.6249012133327131, "naucs_at_1_std": 0.27568453349449135, "naucs_at_1_diff1": 0.851971084561387, "naucs_at_3_max": 0.7154528478057898, "naucs_at_3_std": 0.3263664440135032, "naucs_at_3_diff1": 0.8540364863894296, "naucs_at_5_max": 0.7756769374416456, "naucs_at_5_std": 0.5388525780682669, "naucs_at_5_diff1": 0.8674914410208527, "naucs_at_10_max": 0.7903403785756714, "naucs_at_10_std": 0.6866564807741266, "naucs_at_10_diff1": 0.832187420422716, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}}