| {"vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.342, "ndcg_at_3": 0.40117, "ndcg_at_5": 0.43197, "ndcg_at_10": 0.45845, "ndcg_at_20": 0.48453, "ndcg_at_100": 0.52117, "ndcg_at_1000": 0.53452, "map_at_1": 0.342, "map_at_3": 0.387, "map_at_5": 0.4038, "map_at_10": 0.4147, "map_at_20": 0.42202, "map_at_100": 0.42702, "map_at_1000": 0.4276, "recall_at_1": 0.342, "recall_at_3": 0.442, "recall_at_5": 0.518, "recall_at_10": 0.6, "recall_at_20": 0.702, "recall_at_100": 0.9, "recall_at_1000": 1.0, "precision_at_1": 0.342, "precision_at_3": 0.14733, "precision_at_5": 0.1036, "precision_at_10": 0.06, "precision_at_20": 0.0351, "precision_at_100": 0.009, "precision_at_1000": 0.001, "mrr_at_1": 0.342, "mrr_at_3": 0.387, "mrr_at_5": 0.40379999999999994, "mrr_at_10": 0.41469920634920626, "mrr_at_20": 0.4220213510689516, "mrr_at_100": 0.42702038597455727, "mrr_at_1000": 0.42760577274169886, "naucs_at_1_max": 0.3512734376861317, "naucs_at_1_std": 0.0444730342521779, "naucs_at_1_diff1": 0.6011695349126552, "naucs_at_3_max": 0.30320964696234026, "naucs_at_3_std": 0.05480611266803699, "naucs_at_3_diff1": 0.5116786853219177, "naucs_at_5_max": 0.2763318284117964, "naucs_at_5_std": 0.04854661501247447, "naucs_at_5_diff1": 0.48032697429326143, "naucs_at_10_max": 0.2613713268032054, "naucs_at_10_std": 0.052876224398931505, "naucs_at_10_diff1": 0.42130008904719524, "naucs_at_20_max": 0.2631311978492988, "naucs_at_20_std": 0.0865932616674463, "naucs_at_20_diff1": 0.3475433238044511, "naucs_at_100_max": 0.3759663865546229, "naucs_at_100_std": 0.37533146591970107, "naucs_at_100_diff1": 0.3469934640522881, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.21286, "ndcg_at_3": 0.28038, "ndcg_at_5": 0.30328, "ndcg_at_10": 0.32715, "ndcg_at_20": 0.34879, "ndcg_at_100": 0.38763, "ndcg_at_1000": 0.41855, "map_at_1": 0.21286, "map_at_3": 0.26386, "map_at_5": 0.27661, "map_at_10": 0.28658, "map_at_20": 0.29241, "map_at_100": 0.29738, "map_at_1000": 0.29853, "recall_at_1": 0.21286, "recall_at_3": 0.32816, "recall_at_5": 0.38359, "recall_at_10": 0.45676, "recall_at_20": 0.54324, "recall_at_100": 0.75831, "recall_at_1000": 1.0, "precision_at_1": 0.21286, "precision_at_3": 0.10939, "precision_at_5": 0.07672, "precision_at_10": 0.04568, "precision_at_20": 0.02716, "precision_at_100": 0.00758, "precision_at_1000": 0.001, "mrr_at_1": 0.21286031042128603, "mrr_at_3": 0.26385809312638575, "mrr_at_5": 0.2766075388026607, "mrr_at_10": 0.28658184633794387, "mrr_at_20": 0.29240665519909104, "mrr_at_100": 0.29737825006180746, "mrr_at_1000": 0.29852278499254825, "naucs_at_1_max": 0.1559569059640342, "naucs_at_1_std": 0.05557845139908066, "naucs_at_1_diff1": 0.49407468351994943, "naucs_at_3_max": 0.20679261575387, "naucs_at_3_std": 0.1478983914107042, "naucs_at_3_diff1": 0.4265709856760711, "naucs_at_5_max": 0.19920725280733995, "naucs_at_5_std": 0.17645586905028368, "naucs_at_5_diff1": 0.38320105155993434, "naucs_at_10_max": 0.15292927776389506, "naucs_at_10_std": 0.15367076529659768, "naucs_at_10_diff1": 0.3501748402796659, "naucs_at_20_max": 0.0813370923562777, "naucs_at_20_std": 0.12405275505454354, "naucs_at_20_diff1": 0.301325355523562, "naucs_at_100_max": 0.07220474213528447, "naucs_at_100_std": 0.24916235379027696, "naucs_at_100_diff1": 0.34046058197787155, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.51822, "ndcg_at_3": 0.61531, "ndcg_at_5": 0.64058, "ndcg_at_10": 0.6702, "ndcg_at_20": 0.68117, "ndcg_at_100": 0.6961, "ndcg_at_1000": 0.70177, "map_at_1": 0.51822, "map_at_3": 0.59211, "map_at_5": 0.60628, "map_at_10": 0.61859, "map_at_20": 0.62172, "map_at_100": 0.62377, "map_at_1000": 0.62402, "recall_at_1": 0.51822, "recall_at_3": 0.68219, "recall_at_5": 0.74291, "recall_at_10": 0.83401, "recall_at_20": 0.87652, "recall_at_100": 0.95749, "recall_at_1000": 1.0, "precision_at_1": 0.51822, "precision_at_3": 0.2274, "precision_at_5": 0.14858, "precision_at_10": 0.0834, "precision_at_20": 0.04383, "precision_at_100": 0.00957, "precision_at_1000": 0.001, "mrr_at_1": 0.5182186234817814, "mrr_at_3": 0.5921052631578947, "mrr_at_5": 0.6062753036437245, "mrr_at_10": 0.6185897435897434, "mrr_at_20": 0.6217212657775729, "mrr_at_100": 0.6237693270904571, "mrr_at_1000": 0.6240071088816738, "naucs_at_1_max": 0.3341265251816127, "naucs_at_1_std": -0.11959353621955125, "naucs_at_1_diff1": 0.6910302681103141, "naucs_at_3_max": 0.3399323123068333, "naucs_at_3_std": -0.09657939373539141, "naucs_at_3_diff1": 0.5717390363056882, "naucs_at_5_max": 0.3552696730963818, "naucs_at_5_std": -0.10260460792422756, "naucs_at_5_diff1": 0.5535498159575624, "naucs_at_10_max": 0.31001561070441247, "naucs_at_10_std": -0.07903893227779028, "naucs_at_10_diff1": 0.5078394417215368, "naucs_at_20_max": 0.4489435135781171, "naucs_at_20_std": 0.11789481855251599, "naucs_at_20_diff1": 0.5533054009212981, "naucs_at_100_max": 0.4243960797098085, "naucs_at_100_std": 0.23382054439588018, "naucs_at_100_diff1": 0.6469133129584974, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.46071, "ndcg_at_3": 0.56458, "ndcg_at_5": 0.58132, "ndcg_at_10": 0.61657, "ndcg_at_20": 0.63154, "ndcg_at_100": 0.66086, "ndcg_at_1000": 0.66086, "map_at_1": 0.46071, "map_at_3": 0.53988, "map_at_5": 0.54863, "map_at_10": 0.56354, "map_at_20": 0.56745, "map_at_100": 0.57176, "map_at_1000": 0.57176, "recall_at_1": 0.46071, "recall_at_3": 0.63571, "recall_at_5": 0.67857, "recall_at_10": 0.78571, "recall_at_20": 0.84643, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.46071, "precision_at_3": 0.2119, "precision_at_5": 0.13571, "precision_at_10": 0.07857, "precision_at_20": 0.04232, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.4607142857142857, "mrr_at_3": 0.5398809523809524, "mrr_at_5": 0.5486309523809522, "mrr_at_10": 0.5635445011337866, "mrr_at_20": 0.5674456651756519, "mrr_at_100": 0.571762908944892, "mrr_at_1000": 0.571762908944892, "naucs_at_1_max": 0.507034121474138, "naucs_at_1_std": 0.2780132655504076, "naucs_at_1_diff1": 0.7296146606394313, "naucs_at_3_max": 0.5418566240935775, "naucs_at_3_std": 0.37713949568424254, "naucs_at_3_diff1": 0.6505367829224966, "naucs_at_5_max": 0.5420978160565985, "naucs_at_5_std": 0.40314980006151985, "naucs_at_5_diff1": 0.679310981236543, "naucs_at_10_max": 0.4992307692307692, "naucs_at_10_std": 0.41861369399831, "naucs_at_10_diff1": 0.6698732037193582, "naucs_at_20_max": 0.49946533797465614, "naucs_at_20_std": 0.5021272723136083, "naucs_at_20_diff1": 0.6212317703000932, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/tatdqa_test": {"ndcg_at_1": 0.17861, "ndcg_at_3": 0.23862, "ndcg_at_5": 0.26228, "ndcg_at_10": 0.29096, "ndcg_at_20": 0.31603, "ndcg_at_100": 0.372, "ndcg_at_1000": 0.39451, "map_at_1": 0.17861, "map_at_3": 0.22388, "map_at_5": 0.23694, "map_at_10": 0.24866, "map_at_20": 0.25548, "map_at_100": 0.2628, "map_at_1000": 0.26386, "recall_at_1": 0.17861, "recall_at_3": 0.28129, "recall_at_5": 0.339, "recall_at_10": 0.42831, "recall_at_20": 0.52795, "recall_at_100": 0.83536, "recall_at_1000": 1.0, "precision_at_1": 0.17861, "precision_at_3": 0.09376, "precision_at_5": 0.0678, "precision_at_10": 0.04283, "precision_at_20": 0.0264, "precision_at_100": 0.00835, "precision_at_1000": 0.001, "mrr_at_1": 0.1798298906439854, "mrr_at_3": 0.22428108545970007, "mrr_at_5": 0.23737343053867924, "mrr_at_10": 0.24921816235607222, "mrr_at_20": 0.25603281195395805, "mrr_at_100": 0.26335170093137455, "mrr_at_1000": 0.26441692514341875, "naucs_at_1_max": 0.1232776965489482, "naucs_at_1_std": 0.003190059370457731, "naucs_at_1_diff1": 0.4563902036008701, "naucs_at_3_max": 0.07532335378042614, "naucs_at_3_std": 0.03623353934863348, "naucs_at_3_diff1": 0.3332270965679851, "naucs_at_5_max": 0.06414031260254315, "naucs_at_5_std": 0.05701695108229425, "naucs_at_5_diff1": 0.3018013753606125, "naucs_at_10_max": 0.04831097592835632, "naucs_at_10_std": 0.06825184073629431, "naucs_at_10_diff1": 0.28580154437644, "naucs_at_20_max": 0.036977209968152853, "naucs_at_20_std": 0.06172072256628125, "naucs_at_20_diff1": 0.2373941265383423, "naucs_at_100_max": 0.04903777342859654, "naucs_at_100_std": 0.15510305695187637, "naucs_at_100_diff1": 0.17423930025345663, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/shiftproject_test": {"ndcg_at_1": 0.13, "ndcg_at_3": 0.17393, "ndcg_at_5": 0.18685, "ndcg_at_10": 0.23336, "ndcg_at_20": 0.24812, "ndcg_at_100": 0.30236, "ndcg_at_1000": 0.33487, "map_at_1": 0.13, "map_at_3": 0.16167, "map_at_5": 0.16917, "map_at_10": 0.18908, "map_at_20": 0.1929, "map_at_100": 0.19992, "map_at_1000": 0.20104, "recall_at_1": 0.13, "recall_at_3": 0.21, "recall_at_5": 0.24, "recall_at_10": 0.38, "recall_at_20": 0.44, "recall_at_100": 0.74, "recall_at_1000": 1.0, "precision_at_1": 0.13, "precision_at_3": 0.07, "precision_at_5": 0.048, "precision_at_10": 0.038, "precision_at_20": 0.022, "precision_at_100": 0.0074, "precision_at_1000": 0.001, "mrr_at_1": 0.13, "mrr_at_3": 0.16166666666666668, "mrr_at_5": 0.1691666666666667, "mrr_at_10": 0.1890753968253968, "mrr_at_20": 0.19290321155452733, "mrr_at_100": 0.19991642166943074, "mrr_at_1000": 0.20103804700170202, "naucs_at_1_max": 0.09339392447001414, "naucs_at_1_std": 0.16491683642631688, "naucs_at_1_diff1": 0.17460992590271088, "naucs_at_3_max": 0.17104533865425145, "naucs_at_3_std": 0.26819959776394775, "naucs_at_3_diff1": 0.1107732750566139, "naucs_at_5_max": 0.08402228612508955, "naucs_at_5_std": 0.23617601246105904, "naucs_at_5_diff1": 0.155493649652528, "naucs_at_10_max": 0.2604553842954797, "naucs_at_10_std": 0.3351311309229847, "naucs_at_10_diff1": 0.14200868141293863, "naucs_at_20_max": 0.16427415566111664, "naucs_at_20_std": 0.3091820222373026, "naucs_at_20_diff1": 0.06926971864070627, "naucs_at_100_max": 0.20339807793682382, "naucs_at_100_std": 0.16262519768054814, "naucs_at_100_diff1": 0.023113417947366464, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.5, "ndcg_at_3": 0.57547, "ndcg_at_5": 0.6254, "ndcg_at_10": 0.65888, "ndcg_at_20": 0.66367, "ndcg_at_100": 0.68026, "ndcg_at_1000": 0.68498, "map_at_1": 0.5, "map_at_3": 0.55667, "map_at_5": 0.58467, "map_at_10": 0.59915, "map_at_20": 0.60033, "map_at_100": 0.60259, "map_at_1000": 0.60272, "recall_at_1": 0.5, "recall_at_3": 0.63, "recall_at_5": 0.75, "recall_at_10": 0.85, "recall_at_20": 0.87, "recall_at_100": 0.96, "recall_at_1000": 1.0, "precision_at_1": 0.5, "precision_at_3": 0.21, "precision_at_5": 0.15, "precision_at_10": 0.085, "precision_at_20": 0.0435, "precision_at_100": 0.0096, "precision_at_1000": 0.001, "mrr_at_1": 0.5, "mrr_at_3": 0.5566666666666668, "mrr_at_5": 0.5846666666666667, "mrr_at_10": 0.5991507936507937, "mrr_at_20": 0.600327264239029, "mrr_at_100": 0.6025949843565614, "mrr_at_1000": 0.6027260502596975, "naucs_at_1_max": 0.35581612258494333, "naucs_at_1_std": -0.1489806795469687, "naucs_at_1_diff1": 0.5035842771485672, "naucs_at_3_max": 0.41023833027781703, "naucs_at_3_std": -0.1778467227628131, "naucs_at_3_diff1": 0.45563233519798246, "naucs_at_5_max": 0.44146520146520196, "naucs_at_5_std": -0.19413919413919292, "naucs_at_5_diff1": 0.41034013605442166, "naucs_at_10_max": 0.6098926130816794, "naucs_at_10_std": -0.2327041978522604, "naucs_at_10_diff1": 0.38841522941750706, "naucs_at_20_max": 0.6558542546100861, "naucs_at_20_std": 0.011219728949123188, "naucs_at_20_diff1": 0.3851736651114565, "naucs_at_100_max": 0.6679505135387499, "naucs_at_100_std": 0.2048319327731078, "naucs_at_100_diff1": 0.487861811391226, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.51, "ndcg_at_3": 0.63202, "ndcg_at_5": 0.65655, "ndcg_at_10": 0.67358, "ndcg_at_20": 0.68931, "ndcg_at_100": 0.70649, "ndcg_at_1000": 0.70923, "map_at_1": 0.51, "map_at_3": 0.60167, "map_at_5": 0.61517, "map_at_10": 0.62271, "map_at_20": 0.62731, "map_at_100": 0.62986, "map_at_1000": 0.62999, "recall_at_1": 0.51, "recall_at_3": 0.72, "recall_at_5": 0.78, "recall_at_10": 0.83, "recall_at_20": 0.89, "recall_at_100": 0.98, "recall_at_1000": 1.0, "precision_at_1": 0.51, "precision_at_3": 0.24, "precision_at_5": 0.156, "precision_at_10": 0.083, "precision_at_20": 0.0445, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.51, "mrr_at_3": 0.6016666666666667, "mrr_at_5": 0.6151666666666668, "mrr_at_10": 0.6227063492063494, "mrr_at_20": 0.6273132423132425, "mrr_at_100": 0.6298574595885167, "mrr_at_1000": 0.6299875860892016, "naucs_at_1_max": 0.23866833595842946, "naucs_at_1_std": -0.2843315428832978, "naucs_at_1_diff1": 0.6942503668403266, "naucs_at_3_max": 0.2738095238095245, "naucs_at_3_std": -0.34558150183150077, "naucs_at_3_diff1": 0.6129235347985352, "naucs_at_5_max": 0.32960077896786827, "naucs_at_5_std": -0.32410627347336035, "naucs_at_5_diff1": 0.5276579960124266, "naucs_at_10_max": 0.49669124049457225, "naucs_at_10_std": -0.1449468857026745, "naucs_at_10_diff1": 0.4654031462239507, "naucs_at_20_max": 0.44858689116055367, "naucs_at_20_std": -0.20831543681814282, "naucs_at_20_diff1": 0.3866506313890562, "naucs_at_100_max": 0.5541549953314752, "naucs_at_100_std": -0.8576097105508813, "naucs_at_100_diff1": 0.795751633986937, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.47, "ndcg_at_3": 0.61988, "ndcg_at_5": 0.66119, "ndcg_at_10": 0.69388, "ndcg_at_20": 0.69894, "ndcg_at_100": 0.707, "ndcg_at_1000": 0.7096, "map_at_1": 0.47, "map_at_3": 0.585, "map_at_5": 0.608, "map_at_10": 0.62169, "map_at_20": 0.62308, "map_at_100": 0.62448, "map_at_1000": 0.62458, "recall_at_1": 0.47, "recall_at_3": 0.72, "recall_at_5": 0.82, "recall_at_10": 0.92, "recall_at_20": 0.94, "recall_at_100": 0.98, "recall_at_1000": 1.0, "precision_at_1": 0.47, "precision_at_3": 0.24, "precision_at_5": 0.164, "precision_at_10": 0.092, "precision_at_20": 0.047, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.47, "mrr_at_3": 0.5850000000000001, "mrr_at_5": 0.6080000000000001, "mrr_at_10": 0.6216904761904762, "mrr_at_20": 0.6230793650793651, "mrr_at_100": 0.6244823295062427, "mrr_at_1000": 0.6245820192589574, "naucs_at_1_max": 0.41337570379356287, "naucs_at_1_std": 0.035799522673031194, "naucs_at_1_diff1": 0.6218304566252193, "naucs_at_3_max": 0.5667544261294257, "naucs_at_3_std": 0.1108821733821741, "naucs_at_3_diff1": 0.41882249694749774, "naucs_at_5_max": 0.5240114550060584, "naucs_at_5_std": 0.17796563498182638, "naucs_at_5_diff1": 0.3217590042956274, "naucs_at_10_max": 0.6557539682539693, "naucs_at_10_std": 0.4582749766573311, "naucs_at_10_diff1": 0.3688141923436056, "naucs_at_20_max": 0.6896202925614692, "naucs_at_20_std": 0.5308901338313118, "naucs_at_20_diff1": 0.4508247743541881, "naucs_at_100_max": 0.7117180205415448, "naucs_at_100_std": 0.41433239962652313, "naucs_at_100_diff1": 0.4960317460317565, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.65, "ndcg_at_3": 0.76571, "ndcg_at_5": 0.79068, "ndcg_at_10": 0.80058, "ndcg_at_20": 0.806, "ndcg_at_100": 0.80934, "ndcg_at_1000": 0.81206, "map_at_1": 0.65, "map_at_3": 0.73667, "map_at_5": 0.75067, "map_at_10": 0.75487, "map_at_20": 0.75655, "map_at_100": 0.75688, "map_at_1000": 0.75701, "recall_at_1": 0.65, "recall_at_3": 0.85, "recall_at_5": 0.91, "recall_at_10": 0.94, "recall_at_20": 0.96, "recall_at_100": 0.98, "recall_at_1000": 1.0, "precision_at_1": 0.65, "precision_at_3": 0.28333, "precision_at_5": 0.182, "precision_at_10": 0.094, "precision_at_20": 0.048, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.65, "mrr_at_3": 0.7366666666666668, "mrr_at_5": 0.7506666666666669, "mrr_at_10": 0.754873015873016, "mrr_at_20": 0.7565513375513376, "mrr_at_100": 0.756880382578552, "mrr_at_1000": 0.7570125210427938, "naucs_at_1_max": 0.029408414028805986, "naucs_at_1_std": -0.27022540483359103, "naucs_at_1_diff1": 0.2583285865408084, "naucs_at_3_max": -0.03618613732508907, "naucs_at_3_std": -0.554799869834036, "naucs_at_3_diff1": 0.2528799219004226, "naucs_at_5_max": 0.3330739703288709, "naucs_at_5_std": -0.49128540305010626, "naucs_at_5_diff1": 0.6486150015561787, "naucs_at_10_max": 0.35628695922813397, "naucs_at_10_std": -0.6741363211951432, "naucs_at_10_diff1": 0.6215375038904453, "naucs_at_20_max": 0.6050420168067251, "naucs_at_20_std": -0.141223155929034, "naucs_at_20_diff1": 0.4323062558356693, "naucs_at_100_max": 0.795751633986937, "naucs_at_100_std": 0.1528944911297927, "naucs_at_100_diff1": -0.004668534080297299, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |