| {"tatdqa_test": {"ndcg_at_1": 0.63791, "ndcg_at_3": 0.72965, "ndcg_at_5": 0.75898, "ndcg_at_10": 0.77811, "ndcg_at_20": 0.78611, "ndcg_at_100": 0.79262, "ndcg_at_1000": 0.79403, "map_at_1": 0.63791, "map_at_3": 0.70717, "map_at_5": 0.72348, "map_at_10": 0.73141, "map_at_20": 0.73371, "map_at_100": 0.73468, "map_at_1000": 0.73475, "recall_at_1": 0.63791, "recall_at_3": 0.79465, "recall_at_5": 0.86574, "recall_at_10": 0.92467, "recall_at_20": 0.95565, "recall_at_100": 0.98967, "recall_at_1000": 1.0, "precision_at_1": 0.63791, "precision_at_3": 0.26488, "precision_at_5": 0.17315, "precision_at_10": 0.09247, "precision_at_20": 0.04778, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.637910085054678, "mrr_at_3": 0.7091940056703118, "mrr_at_5": 0.7245949777237748, "mrr_at_10": 0.7321893864876854, "mrr_at_20": 0.734534735983059, "mrr_at_100": 0.7354492680936119, "mrr_at_1000": 0.7355095493431769, "naucs_at_1_max": 0.11706806975981984, "naucs_at_1_std": -0.3409555849470442, "naucs_at_1_diff1": 0.7715117941189259, "naucs_at_3_max": 0.14662775923175694, "naucs_at_3_std": -0.36798133689268664, "naucs_at_3_diff1": 0.6848741632409352, "naucs_at_5_max": 0.224912526144752, "naucs_at_5_std": -0.3125954444862817, "naucs_at_5_diff1": 0.6637427209678819, "naucs_at_10_max": 0.30806849861185404, "naucs_at_10_std": -0.03316507855697272, "naucs_at_10_diff1": 0.557511941932588, "naucs_at_20_max": 0.33123123923265707, "naucs_at_20_std": 0.15688742820732718, "naucs_at_20_diff1": 0.5492605591760332, "naucs_at_100_max": 0.5543174846455237, "naucs_at_100_std": 0.5976827798155245, "naucs_at_100_diff1": 0.614203438126567, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shiftproject_test": {"ndcg_at_1": 0.74, "ndcg_at_3": 0.85571, "ndcg_at_5": 0.85958, "ndcg_at_10": 0.8656, "ndcg_at_20": 0.86805, "ndcg_at_100": 0.87169, "ndcg_at_1000": 0.87169, "map_at_1": 0.74, "map_at_3": 0.82667, "map_at_5": 0.82867, "map_at_10": 0.83089, "map_at_20": 0.83151, "map_at_100": 0.83197, "map_at_1000": 0.83197, "recall_at_1": 0.74, "recall_at_3": 0.94, "recall_at_5": 0.95, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.74, "precision_at_3": 0.31333, "precision_at_5": 0.19, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.76, "mrr_at_3": 0.84, "mrr_at_5": 0.8420000000000001, "mrr_at_10": 0.8442222222222222, "mrr_at_20": 0.8448472222222222, "mrr_at_100": 0.84531461352657, "mrr_at_1000": 0.84531461352657, "naucs_at_1_max": -0.10372653177081258, "naucs_at_1_std": -0.3416933619885646, "naucs_at_1_diff1": 0.6173512834029431, "naucs_at_3_max": -0.11640211640211533, "naucs_at_3_std": -0.039604730781200366, "naucs_at_3_diff1": 0.5161842514783699, "naucs_at_5_max": 0.20830999066293704, "naucs_at_5_std": -0.013258636788043418, "naucs_at_5_diff1": 0.6536881419234374, "naucs_at_10_max": -0.27591036414565706, "naucs_at_10_std": -0.3963585434173691, "naucs_at_10_diff1": 0.807812013694364, "naucs_at_20_max": -0.5929038281979383, "naucs_at_20_std": -0.5088702147525547, "naucs_at_20_diff1": 0.7770774976657324, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.98262, "ndcg_at_5": 0.98693, "ndcg_at_10": 0.98693, "ndcg_at_20": 0.98693, "ndcg_at_100": 0.98693, "ndcg_at_1000": 0.98693, "map_at_1": 0.97, "map_at_3": 0.98, "map_at_5": 0.9825, "map_at_10": 0.9825, "map_at_20": 0.9825, "map_at_100": 0.9825, "map_at_1000": 0.9825, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.97, "mrr_at_3": 0.98, "mrr_at_5": 0.9825, "mrr_at_10": 0.9825, "mrr_at_20": 0.9825, "mrr_at_100": 0.9825, "mrr_at_1000": 0.9825, "naucs_at_1_max": 0.8078120136943662, "naucs_at_1_std": 0.08839091192032505, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 0.8692810457516356, "naucs_at_3_std": -0.1713352007469878, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.84, "ndcg_at_3": 0.9194, "ndcg_at_5": 0.92802, "ndcg_at_10": 0.92802, "ndcg_at_20": 0.93046, "ndcg_at_100": 0.93046, "ndcg_at_1000": 0.93046, "map_at_1": 0.84, "map_at_3": 0.90167, "map_at_5": 0.90667, "map_at_10": 0.90667, "map_at_20": 0.90729, "map_at_100": 0.90729, "map_at_1000": 0.90729, "recall_at_1": 0.84, "recall_at_3": 0.97, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.84, "precision_at_3": 0.32333, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.86, "mrr_at_3": 0.9166666666666667, "mrr_at_5": 0.9191666666666667, "mrr_at_10": 0.9191666666666667, "mrr_at_20": 0.9198333333333334, "mrr_at_100": 0.9198333333333334, "mrr_at_1000": 0.9198333333333334, "naucs_at_1_max": 0.5329528718703975, "naucs_at_1_std": 0.33511291114383884, "naucs_at_1_diff1": 0.9298907707412855, "naucs_at_3_max": 0.807812013694371, "naucs_at_3_std": 0.8202614379084949, "naucs_at_3_diff1": 0.9128540305010931, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.7222222222222276, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.7222222222222276, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "arxivqa_test_subsampled": {"ndcg_at_1": 0.804, "ndcg_at_3": 0.85212, "ndcg_at_5": 0.86142, "ndcg_at_10": 0.87345, "ndcg_at_20": 0.87914, "ndcg_at_100": 0.88443, "ndcg_at_1000": 0.88558, "map_at_1": 0.804, "map_at_3": 0.841, "map_at_5": 0.8463, "map_at_10": 0.8515, "map_at_20": 0.85312, "map_at_100": 0.85389, "map_at_1000": 0.85396, "recall_at_1": 0.804, "recall_at_3": 0.884, "recall_at_5": 0.906, "recall_at_10": 0.942, "recall_at_20": 0.964, "recall_at_100": 0.992, "recall_at_1000": 1.0, "precision_at_1": 0.804, "precision_at_3": 0.29467, "precision_at_5": 0.1812, "precision_at_10": 0.0942, "precision_at_20": 0.0482, "precision_at_100": 0.00992, "precision_at_1000": 0.001, "mrr_at_1": 0.798, "mrr_at_3": 0.8366666666666667, "mrr_at_5": 0.8428666666666667, "mrr_at_10": 0.847368253968254, "mrr_at_20": 0.849015823065823, "mrr_at_100": 0.8497704886379664, "mrr_at_1000": 0.8498371151009344, "naucs_at_1_max": 0.7272185915837046, "naucs_at_1_std": 0.09151753903784711, "naucs_at_1_diff1": 0.9220260998798836, "naucs_at_3_max": 0.7894111282337131, "naucs_at_3_std": 0.12989770156398434, "naucs_at_3_diff1": 0.907660906914981, "naucs_at_5_max": 0.7784830244154408, "naucs_at_5_std": 0.160041321493135, "naucs_at_5_diff1": 0.8836839700419182, "naucs_at_10_max": 0.8039537654142149, "naucs_at_10_std": 0.1776940661322011, "naucs_at_10_diff1": 0.8972922502334268, "naucs_at_20_max": 0.9162257495590788, "naucs_at_20_std": 0.3446156240273899, "naucs_at_20_diff1": 0.9183006535947664, "naucs_at_100_max": 0.9346405228758211, "naucs_at_100_std": 0.3015873015873097, "naucs_at_100_diff1": 0.9346405228758211, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_test_subsampled": {"ndcg_at_1": 0.51663, "ndcg_at_3": 0.58497, "ndcg_at_5": 0.61407, "ndcg_at_10": 0.63327, "ndcg_at_20": 0.64235, "ndcg_at_100": 0.66195, "ndcg_at_1000": 0.67371, "map_at_1": 0.51663, "map_at_3": 0.568, "map_at_5": 0.58407, "map_at_10": 0.5919, "map_at_20": 0.59446, "map_at_100": 0.59696, "map_at_1000": 0.59742, "recall_at_1": 0.51663, "recall_at_3": 0.63415, "recall_at_5": 0.7051, "recall_at_10": 0.76497, "recall_at_20": 0.80044, "recall_at_100": 0.90909, "recall_at_1000": 1.0, "precision_at_1": 0.51663, "precision_at_3": 0.21138, "precision_at_5": 0.14102, "precision_at_10": 0.0765, "precision_at_20": 0.04002, "precision_at_100": 0.00909, "precision_at_1000": 0.001, "mrr_at_1": 0.5121951219512195, "mrr_at_3": 0.5628233555062824, "mrr_at_5": 0.5796747967479675, "mrr_at_10": 0.5882078625981065, "mrr_at_20": 0.590892152282475, "mrr_at_100": 0.5933575456979939, "mrr_at_1000": 0.5937484864645279, "naucs_at_1_max": 0.5535805693462823, "naucs_at_1_std": 0.29052195068342307, "naucs_at_1_diff1": 0.8498842552231669, "naucs_at_3_max": 0.5130075041166375, "naucs_at_3_std": 0.40344135721957186, "naucs_at_3_diff1": 0.7530683160559333, "naucs_at_5_max": 0.5036801822293181, "naucs_at_5_std": 0.44055761888976297, "naucs_at_5_diff1": 0.716299635236424, "naucs_at_10_max": 0.4530835967070493, "naucs_at_10_std": 0.4884234567947281, "naucs_at_10_diff1": 0.7157109941613901, "naucs_at_20_max": 0.41878776502178355, "naucs_at_20_std": 0.5289048480486104, "naucs_at_20_diff1": 0.694664426657451, "naucs_at_100_max": 0.5417887784952314, "naucs_at_100_std": 0.8428413439955713, "naucs_at_100_diff1": 0.5928734870103426, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.95, "ndcg_at_3": 0.98024, "ndcg_at_5": 0.98024, "ndcg_at_10": 0.98024, "ndcg_at_20": 0.98024, "ndcg_at_100": 0.98024, "ndcg_at_1000": 0.98024, "map_at_1": 0.95, "map_at_3": 0.97333, "map_at_5": 0.97333, "map_at_10": 0.97333, "map_at_20": 0.97333, "map_at_100": 0.97333, "map_at_1000": 0.97333, "recall_at_1": 0.95, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.95, "mrr_at_3": 0.9733333333333333, "mrr_at_5": 0.9733333333333333, "mrr_at_10": 0.9733333333333333, "mrr_at_20": 0.9733333333333333, "mrr_at_100": 0.9733333333333333, "mrr_at_1000": 0.9733333333333333, "naucs_at_1_max": 0.7343604108309969, "naucs_at_1_std": 0.2407096171802029, "naucs_at_1_diff1": 0.9477124183006508, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_test_subsampled": {"ndcg_at_1": 0.86032, "ndcg_at_3": 0.89529, "ndcg_at_5": 0.906, "ndcg_at_10": 0.90853, "ndcg_at_20": 0.91538, "ndcg_at_100": 0.91786, "ndcg_at_1000": 0.91884, "map_at_1": 0.86032, "map_at_3": 0.88765, "map_at_5": 0.89352, "map_at_10": 0.89451, "map_at_20": 0.89649, "map_at_100": 0.89678, "map_at_1000": 0.8968, "recall_at_1": 0.86032, "recall_at_3": 0.917, "recall_at_5": 0.94332, "recall_at_10": 0.95142, "recall_at_20": 0.97773, "recall_at_100": 0.9919, "recall_at_1000": 1.0, "precision_at_1": 0.86032, "precision_at_3": 0.30567, "precision_at_5": 0.18866, "precision_at_10": 0.09514, "precision_at_20": 0.04889, "precision_at_100": 0.00992, "precision_at_1000": 0.001, "mrr_at_1": 0.8603238866396761, "mrr_at_3": 0.886302294197031, "mrr_at_5": 0.8926788124156545, "mrr_at_10": 0.8938226977700662, "mrr_at_20": 0.8955678026096578, "mrr_at_100": 0.8958468497410749, "mrr_at_1000": 0.8958752640092609, "naucs_at_1_max": 0.5508889548870433, "naucs_at_1_std": -0.09355736121967415, "naucs_at_1_diff1": 0.9383378498497967, "naucs_at_3_max": 0.688609278197004, "naucs_at_3_std": 0.008706783843416104, "naucs_at_3_diff1": 0.8903413884162533, "naucs_at_5_max": 0.8051659664645568, "naucs_at_5_std": 0.14011613476580845, "naucs_at_5_diff1": 0.9136802324400093, "naucs_at_10_max": 0.8332623194529386, "naucs_at_10_std": 0.20617039762215936, "naucs_at_10_diff1": 0.9217493397586589, "naucs_at_20_max": 0.9223599083831424, "naucs_at_20_std": 0.47630555763007465, "naucs_at_20_diff1": 0.927259949634811, "naucs_at_100_max": 0.8979159520443043, "naucs_at_100_std": 0.7806644160411462, "naucs_at_100_diff1": 0.9346992729676393, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.94893, "ndcg_at_5": 0.94893, "ndcg_at_10": 0.95509, "ndcg_at_20": 0.95754, "ndcg_at_100": 0.95754, "ndcg_at_1000": 0.95754, "map_at_1": 0.92, "map_at_3": 0.94167, "map_at_5": 0.94167, "map_at_10": 0.94403, "map_at_20": 0.94465, "map_at_100": 0.94465, "map_at_1000": 0.94465, "recall_at_1": 0.92, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.92, "mrr_at_3": 0.9416666666666668, "mrr_at_5": 0.9416666666666668, "mrr_at_10": 0.9445238095238095, "mrr_at_20": 0.9451904761904762, "mrr_at_100": 0.9451904761904762, "mrr_at_1000": 0.9451904761904762, "naucs_at_1_max": 0.5991479925303451, "naucs_at_1_std": -0.49241363211951594, "naucs_at_1_diff1": 0.9652777777777789, "naucs_at_3_max": 0.7642390289449176, "naucs_at_3_std": -0.8249299719887898, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7642390289449118, "naucs_at_5_std": -0.8249299719887969, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.8692810457516413, "naucs_at_10_std": -0.1713352007469681, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.825, "ndcg_at_3": 0.88125, "ndcg_at_5": 0.89032, "ndcg_at_10": 0.89812, "ndcg_at_20": 0.90175, "ndcg_at_100": 0.90615, "ndcg_at_1000": 0.90615, "map_at_1": 0.825, "map_at_3": 0.86845, "map_at_5": 0.87363, "map_at_10": 0.87667, "map_at_20": 0.87768, "map_at_100": 0.87846, "map_at_1000": 0.87846, "recall_at_1": 0.825, "recall_at_3": 0.91786, "recall_at_5": 0.93929, "recall_at_10": 0.96429, "recall_at_20": 0.97857, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.825, "precision_at_3": 0.30595, "precision_at_5": 0.18786, "precision_at_10": 0.09643, "precision_at_20": 0.04893, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.8214285714285714, "mrr_at_3": 0.8660714285714286, "mrr_at_5": 0.87125, "mrr_at_10": 0.8741907596371882, "mrr_at_20": 0.8752038353220075, "mrr_at_100": 0.8759897415616305, "mrr_at_1000": 0.8759897415616305, "naucs_at_1_max": 0.3952340580148981, "naucs_at_1_std": 0.12516401219241538, "naucs_at_1_diff1": 0.8465148670744261, "naucs_at_3_max": 0.5806438517435943, "naucs_at_3_std": 0.3494905208460206, "naucs_at_3_diff1": 0.818454918199163, "naucs_at_5_max": 0.5258417092327137, "naucs_at_5_std": 0.25992750041192964, "naucs_at_5_diff1": 0.7860987532267822, "naucs_at_10_max": 0.8415499533146577, "naucs_at_10_std": 0.7990662931839378, "naucs_at_10_diff1": 0.8204948646125075, "naucs_at_20_max": 0.8821195144724577, "naucs_at_20_std": 0.8330999066293233, "naucs_at_20_diff1": 0.7226112667289174, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |