| {"vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.194, "ndcg_at_3": 0.23428, "ndcg_at_5": 0.25399, "ndcg_at_10": 0.27746, "ndcg_at_20": 0.29815, "ndcg_at_100": 0.34405, "ndcg_at_1000": 0.38082, "map_at_1": 0.194, "map_at_3": 0.22467, "map_at_5": 0.23557, "map_at_10": 0.24536, "map_at_20": 0.25102, "map_at_100": 0.25699, "map_at_1000": 0.25845, "recall_at_1": 0.194, "recall_at_3": 0.262, "recall_at_5": 0.31, "recall_at_10": 0.382, "recall_at_20": 0.464, "recall_at_100": 0.718, "recall_at_1000": 1.0, "precision_at_1": 0.194, "precision_at_3": 0.08733, "precision_at_5": 0.062, "precision_at_10": 0.0382, "precision_at_20": 0.0232, "precision_at_100": 0.00718, "precision_at_1000": 0.001, "mrr_at_1": 0.194, "mrr_at_3": 0.22466666666666665, "mrr_at_5": 0.2355666666666667, "mrr_at_10": 0.2453603174603175, "mrr_at_20": 0.25101822261845486, "mrr_at_100": 0.25698538286605843, "mrr_at_1000": 0.2584512551524216, "naucs_at_1_max": 0.45616095322842626, "naucs_at_1_std": -0.09688090268244365, "naucs_at_1_diff1": 0.49830465863267814, "naucs_at_3_max": 0.3924181183865958, "naucs_at_3_std": -0.10766974824050675, "naucs_at_3_diff1": 0.38430557996295184, "naucs_at_5_max": 0.34086074996302623, "naucs_at_5_std": -0.09092864729687923, "naucs_at_5_diff1": 0.3439208357422323, "naucs_at_10_max": 0.28690953188417473, "naucs_at_10_std": -0.06333469229830402, "naucs_at_10_diff1": 0.27416728772231386, "naucs_at_20_max": 0.2243962903701211, "naucs_at_20_std": -0.030390753294875377, "naucs_at_20_diff1": 0.2522606214998528, "naucs_at_100_max": 0.15254127991442465, "naucs_at_100_std": 0.030546947012256007, "naucs_at_100_diff1": 0.19091367727888314, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.07317, "ndcg_at_3": 0.10717, "ndcg_at_5": 0.119, "ndcg_at_10": 0.13733, "ndcg_at_20": 0.15798, "ndcg_at_100": 0.20904, "ndcg_at_1000": 0.26333, "map_at_1": 0.07317, "map_at_3": 0.0983, "map_at_5": 0.10484, "map_at_10": 0.11222, "map_at_20": 0.11783, "map_at_100": 0.12461, "map_at_1000": 0.12673, "recall_at_1": 0.07317, "recall_at_3": 0.13304, "recall_at_5": 0.16186, "recall_at_10": 0.21951, "recall_at_20": 0.30155, "recall_at_100": 0.58093, "recall_at_1000": 1.0, "precision_at_1": 0.07317, "precision_at_3": 0.04435, "precision_at_5": 0.03237, "precision_at_10": 0.02195, "precision_at_20": 0.01508, "precision_at_100": 0.00581, "precision_at_1000": 0.001, "mrr_at_1": 0.07317073170731707, "mrr_at_3": 0.09830007390983003, "mrr_at_5": 0.10484109386548415, "mrr_at_10": 0.11222415795586532, "mrr_at_20": 0.11783019841513877, "mrr_at_100": 0.12460791891127795, "mrr_at_1000": 0.12673074064268428, "naucs_at_1_max": 0.12025982529820782, "naucs_at_1_std": -0.04761217351657549, "naucs_at_1_diff1": 0.36251599396437884, "naucs_at_3_max": 0.024510873492006768, "naucs_at_3_std": -0.05837029747252124, "naucs_at_3_diff1": 0.22097986209322437, "naucs_at_5_max": 0.03595820524173138, "naucs_at_5_std": -0.04038720990562407, "naucs_at_5_diff1": 0.20800128710951715, "naucs_at_10_max": 0.04002139668192121, "naucs_at_10_std": -0.014030646078733083, "naucs_at_10_diff1": 0.1745591420097609, "naucs_at_20_max": 0.09316008827855647, "naucs_at_20_std": 0.012983126164409945, "naucs_at_20_diff1": 0.17349647375521185, "naucs_at_100_max": 0.09174360315381583, "naucs_at_100_std": 0.07239555875987534, "naucs_at_100_diff1": 0.2057682842033964, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.26518, "ndcg_at_3": 0.32656, "ndcg_at_5": 0.35548, "ndcg_at_10": 0.37851, "ndcg_at_20": 0.4037, "ndcg_at_100": 0.44402, "ndcg_at_1000": 0.46603, "map_at_1": 0.26518, "map_at_3": 0.3114, "map_at_5": 0.32729, "map_at_10": 0.33687, "map_at_20": 0.34358, "map_at_100": 0.34904, "map_at_1000": 0.34993, "recall_at_1": 0.26518, "recall_at_3": 0.37045, "recall_at_5": 0.4413, "recall_at_10": 0.51215, "recall_at_20": 0.61336, "recall_at_100": 0.83198, "recall_at_1000": 1.0, "precision_at_1": 0.26518, "precision_at_3": 0.12348, "precision_at_5": 0.08826, "precision_at_10": 0.05121, "precision_at_20": 0.03067, "precision_at_100": 0.00832, "precision_at_1000": 0.001, "mrr_at_1": 0.2651821862348178, "mrr_at_3": 0.3114035087719298, "mrr_at_5": 0.32729419703103907, "mrr_at_10": 0.33687423687423684, "mrr_at_20": 0.34357647182184037, "mrr_at_100": 0.34903967645027933, "mrr_at_1000": 0.3499322688157548, "naucs_at_1_max": 0.22725907310977225, "naucs_at_1_std": -0.05555633255154067, "naucs_at_1_diff1": 0.46552534956006875, "naucs_at_3_max": 0.262760949249393, "naucs_at_3_std": 0.005926985746904812, "naucs_at_3_diff1": 0.4071992932728183, "naucs_at_5_max": 0.20131834403466364, "naucs_at_5_std": -0.012424200647473432, "naucs_at_5_diff1": 0.3381884916533942, "naucs_at_10_max": 0.20143586047292864, "naucs_at_10_std": 0.03748395663221178, "naucs_at_10_diff1": 0.30830976550271083, "naucs_at_20_max": 0.22773274626635015, "naucs_at_20_std": 0.07353037392687715, "naucs_at_20_diff1": 0.2860181621323394, "naucs_at_100_max": 0.2642917251934298, "naucs_at_100_std": 0.29308298721238213, "naucs_at_100_diff1": 0.2564427320022936, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.125, "ndcg_at_3": 0.17712, "ndcg_at_5": 0.20249, "ndcg_at_10": 0.22523, "ndcg_at_20": 0.26876, "ndcg_at_100": 0.36104, "ndcg_at_1000": 0.36104, "map_at_1": 0.125, "map_at_3": 0.16429, "map_at_5": 0.17857, "map_at_10": 0.18775, "map_at_20": 0.19932, "map_at_100": 0.21325, "map_at_1000": 0.21325, "recall_at_1": 0.125, "recall_at_3": 0.21429, "recall_at_5": 0.275, "recall_at_10": 0.34643, "recall_at_20": 0.52143, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.125, "precision_at_3": 0.07143, "precision_at_5": 0.055, "precision_at_10": 0.03464, "precision_at_20": 0.02607, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.125, "mrr_at_3": 0.1642857142857143, "mrr_at_5": 0.1785714285714286, "mrr_at_10": 0.18775368480725627, "mrr_at_20": 0.19932329128743903, "mrr_at_100": 0.21325209977362394, "mrr_at_1000": 0.21325209977362394, "naucs_at_1_max": 0.09782528263103807, "naucs_at_1_std": 0.10585817060637205, "naucs_at_1_diff1": 0.16069064748201445, "naucs_at_3_max": 0.028766969594973817, "naucs_at_3_std": 0.1354762706159543, "naucs_at_3_diff1": 0.07179961853472462, "naucs_at_5_max": -0.024564969136578685, "naucs_at_5_std": 0.11742689117059667, "naucs_at_5_diff1": 0.020661136236725247, "naucs_at_10_max": -0.01236326586446157, "naucs_at_10_std": 0.12392039585092379, "naucs_at_10_diff1": -0.011745338421407597, "naucs_at_20_max": -0.05765357257538746, "naucs_at_20_std": 0.1448491276552269, "naucs_at_20_diff1": 0.002199106734521382, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/tatdqa_test": {"ndcg_at_1": 0.0158, "ndcg_at_3": 0.02802, "ndcg_at_5": 0.0328, "ndcg_at_10": 0.04118, "ndcg_at_20": 0.05228, "ndcg_at_100": 0.10794, "ndcg_at_1000": 0.18345, "map_at_1": 0.0158, "map_at_3": 0.02491, "map_at_5": 0.02758, "map_at_10": 0.031, "map_at_20": 0.03399, "map_at_100": 0.04051, "map_at_1000": 0.04382, "recall_at_1": 0.0158, "recall_at_3": 0.03706, "recall_at_5": 0.0486, "recall_at_10": 0.07473, "recall_at_20": 0.11908, "recall_at_100": 0.43682, "recall_at_1000": 1.0, "precision_at_1": 0.0158, "precision_at_3": 0.01235, "precision_at_5": 0.00972, "precision_at_10": 0.00747, "precision_at_20": 0.00595, "precision_at_100": 0.00437, "precision_at_1000": 0.001, "mrr_at_1": 0.015795868772782502, "mrr_at_3": 0.024908869987849337, "mrr_at_5": 0.027582017010935603, "mrr_at_10": 0.031003731990973803, "mrr_at_20": 0.03398397803628708, "mrr_at_100": 0.04050990121716738, "mrr_at_1000": 0.04381244666582136, "naucs_at_1_max": 0.37768627847426967, "naucs_at_1_std": 0.08918421070395563, "naucs_at_1_diff1": 0.4668179771401452, "naucs_at_3_max": 0.26401253705627126, "naucs_at_3_std": 0.15143117653948604, "naucs_at_3_diff1": 0.25305680385702695, "naucs_at_5_max": 0.19373179609836738, "naucs_at_5_std": 0.11509578090146863, "naucs_at_5_diff1": 0.21679819158947974, "naucs_at_10_max": 0.1419629098861985, "naucs_at_10_std": 0.08902812942300867, "naucs_at_10_diff1": 0.13005574395144884, "naucs_at_20_max": 0.11012969448282328, "naucs_at_20_std": 0.08844711041409134, "naucs_at_20_diff1": 0.11118707019441564, "naucs_at_100_max": -0.0013902169449798443, "naucs_at_100_std": -0.018452319538778773, "naucs_at_100_diff1": 0.06992819597017841, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/shiftproject_test": {"ndcg_at_1": 0.02, "ndcg_at_3": 0.02631, "ndcg_at_5": 0.03835, "ndcg_at_10": 0.04151, "ndcg_at_20": 0.05132, "ndcg_at_100": 0.0925, "ndcg_at_1000": 0.17178, "map_at_1": 0.02, "map_at_3": 0.025, "map_at_5": 0.0315, "map_at_10": 0.03275, "map_at_20": 0.03527, "map_at_100": 0.04043, "map_at_1000": 0.04278, "recall_at_1": 0.02, "recall_at_3": 0.03, "recall_at_5": 0.06, "recall_at_10": 0.07, "recall_at_20": 0.11, "recall_at_100": 0.34, "recall_at_1000": 1.0, "precision_at_1": 0.02, "precision_at_3": 0.01, "precision_at_5": 0.012, "precision_at_10": 0.007, "precision_at_20": 0.0055, "precision_at_100": 0.0034, "precision_at_1000": 0.001, "mrr_at_1": 0.02, "mrr_at_3": 0.025, "mrr_at_5": 0.0315, "mrr_at_10": 0.03275, "mrr_at_20": 0.035274743230625585, "mrr_at_100": 0.040428839198707794, "mrr_at_1000": 0.04278256202231608, "naucs_at_1_max": 1.0, "naucs_at_1_std": 0.17171988194258103, "naucs_at_1_diff1": 0.8309632412127715, "naucs_at_3_max": 0.5605044271532065, "naucs_at_3_std": -0.07709507199713805, "naucs_at_3_diff1": 0.41561577676415334, "naucs_at_5_max": 0.15092567748859678, "naucs_at_5_std": -0.026384044360969413, "naucs_at_5_diff1": 0.22560593864591733, "naucs_at_10_max": 0.11928398942082882, "naucs_at_10_std": -0.09398597109893045, "naucs_at_10_diff1": 0.18329564184138938, "naucs_at_20_max": -0.10735586481113325, "naucs_at_20_std": -0.13138888189848252, "naucs_at_20_diff1": -0.0019377406447392033, "naucs_at_100_max": 0.2334190095127056, "naucs_at_100_std": 0.05629454940512453, "naucs_at_100_diff1": 0.012798431330983727, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.11, "ndcg_at_3": 0.13893, "ndcg_at_5": 0.15185, "ndcg_at_10": 0.17854, "ndcg_at_20": 0.19105, "ndcg_at_100": 0.22797, "ndcg_at_1000": 0.28744, "map_at_1": 0.11, "map_at_3": 0.13167, "map_at_5": 0.13917, "map_at_10": 0.15069, "map_at_20": 0.15404, "map_at_100": 0.15917, "map_at_1000": 0.16121, "recall_at_1": 0.11, "recall_at_3": 0.16, "recall_at_5": 0.19, "recall_at_10": 0.27, "recall_at_20": 0.32, "recall_at_100": 0.52, "recall_at_1000": 1.0, "precision_at_1": 0.11, "precision_at_3": 0.05333, "precision_at_5": 0.038, "precision_at_10": 0.027, "precision_at_20": 0.016, "precision_at_100": 0.0052, "precision_at_1000": 0.001, "mrr_at_1": 0.11, "mrr_at_3": 0.13166666666666668, "mrr_at_5": 0.1391666666666667, "mrr_at_10": 0.15069047619047618, "mrr_at_20": 0.15404365079365076, "mrr_at_100": 0.1591672338493456, "mrr_at_1000": 0.1612132569973971, "naucs_at_1_max": 0.3742104336009261, "naucs_at_1_std": -0.07793743865918408, "naucs_at_1_diff1": 0.48315172257593664, "naucs_at_3_max": 0.4361461053925333, "naucs_at_3_std": 0.07094023659548324, "naucs_at_3_diff1": 0.38936472576432635, "naucs_at_5_max": 0.41450742353673076, "naucs_at_5_std": 0.1406520605989271, "naucs_at_5_diff1": 0.3028769357523582, "naucs_at_10_max": 0.31593004588785156, "naucs_at_10_std": 0.07330690874994661, "naucs_at_10_diff1": 0.23320405176945244, "naucs_at_20_max": 0.30176335224232054, "naucs_at_20_std": 0.1306726961361445, "naucs_at_20_diff1": 0.2355640586535375, "naucs_at_100_max": 0.3237390500027205, "naucs_at_100_std": 0.16541977256651594, "naucs_at_100_diff1": 0.3194542684585667, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.13, "ndcg_at_3": 0.18024, "ndcg_at_5": 0.19659, "ndcg_at_10": 0.21217, "ndcg_at_20": 0.21951, "ndcg_at_100": 0.27521, "ndcg_at_1000": 0.31987, "map_at_1": 0.13, "map_at_3": 0.16667, "map_at_5": 0.17567, "map_at_10": 0.18175, "map_at_20": 0.18363, "map_at_100": 0.19126, "map_at_1000": 0.19281, "recall_at_1": 0.13, "recall_at_3": 0.22, "recall_at_5": 0.26, "recall_at_10": 0.31, "recall_at_20": 0.34, "recall_at_100": 0.64, "recall_at_1000": 1.0, "precision_at_1": 0.13, "precision_at_3": 0.07333, "precision_at_5": 0.052, "precision_at_10": 0.031, "precision_at_20": 0.017, "precision_at_100": 0.0064, "precision_at_1000": 0.001, "mrr_at_1": 0.13, "mrr_at_3": 0.16666666666666669, "mrr_at_5": 0.17566666666666667, "mrr_at_10": 0.18174603174603174, "mrr_at_20": 0.18362593370681604, "mrr_at_100": 0.1912619264204802, "mrr_at_1000": 0.19284749551243116, "naucs_at_1_max": 0.3078059591233686, "naucs_at_1_std": 0.0830964160193412, "naucs_at_1_diff1": 0.5082044279286339, "naucs_at_3_max": 0.3818099885011033, "naucs_at_3_std": 0.12728035553345576, "naucs_at_3_diff1": 0.328573204462815, "naucs_at_5_max": 0.28138822236998073, "naucs_at_5_std": 0.0749761498655718, "naucs_at_5_diff1": 0.21800468329912404, "naucs_at_10_max": 0.20995737929736316, "naucs_at_10_std": 0.04539037605711421, "naucs_at_10_diff1": 0.14722293182032062, "naucs_at_20_max": 0.21773231935133416, "naucs_at_20_std": 0.08542886668963152, "naucs_at_20_diff1": 0.17413021012745442, "naucs_at_100_max": 0.2542789628305023, "naucs_at_100_std": 0.33551539149316634, "naucs_at_100_diff1": 0.24057989526120854, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.15, "ndcg_at_3": 0.20155, "ndcg_at_5": 0.21403, "ndcg_at_10": 0.22048, "ndcg_at_20": 0.23306, "ndcg_at_100": 0.2746, "ndcg_at_1000": 0.32774, "map_at_1": 0.15, "map_at_3": 0.18833, "map_at_5": 0.19533, "map_at_10": 0.198, "map_at_20": 0.20142, "map_at_100": 0.20675, "map_at_1000": 0.2085, "recall_at_1": 0.15, "recall_at_3": 0.24, "recall_at_5": 0.27, "recall_at_10": 0.29, "recall_at_20": 0.34, "recall_at_100": 0.57, "recall_at_1000": 1.0, "precision_at_1": 0.15, "precision_at_3": 0.08, "precision_at_5": 0.054, "precision_at_10": 0.029, "precision_at_20": 0.017, "precision_at_100": 0.0057, "precision_at_1000": 0.001, "mrr_at_1": 0.15, "mrr_at_3": 0.18833333333333335, "mrr_at_5": 0.1953333333333333, "mrr_at_10": 0.19799999999999993, "mrr_at_20": 0.20141953368269153, "mrr_at_100": 0.20674683837103697, "mrr_at_1000": 0.20849665101112883, "naucs_at_1_max": 0.4102207438766253, "naucs_at_1_std": 0.15566979135167824, "naucs_at_1_diff1": 0.38482007862110673, "naucs_at_3_max": 0.4429367361610353, "naucs_at_3_std": 0.1598070932183081, "naucs_at_3_diff1": 0.3894680086268872, "naucs_at_5_max": 0.4120814331784793, "naucs_at_5_std": 0.16141726690249875, "naucs_at_5_diff1": 0.3698163065251671, "naucs_at_10_max": 0.3740367841984531, "naucs_at_10_std": 0.16437509443292161, "naucs_at_10_diff1": 0.32836558933011006, "naucs_at_20_max": 0.3745197276027454, "naucs_at_20_std": 0.1611595431781447, "naucs_at_20_diff1": 0.3420731868888948, "naucs_at_100_max": 0.2898494568103216, "naucs_at_100_std": 0.25054175456724226, "naucs_at_100_diff1": 0.05269729194471972, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.17, "ndcg_at_3": 0.19524, "ndcg_at_5": 0.20772, "ndcg_at_10": 0.21374, "ndcg_at_20": 0.22103, "ndcg_at_100": 0.2848, "ndcg_at_1000": 0.32712, "map_at_1": 0.17, "map_at_3": 0.19, "map_at_5": 0.197, "map_at_10": 0.19922, "map_at_20": 0.20108, "map_at_100": 0.20869, "map_at_1000": 0.20996, "recall_at_1": 0.17, "recall_at_3": 0.21, "recall_at_5": 0.24, "recall_at_10": 0.26, "recall_at_20": 0.29, "recall_at_100": 0.65, "recall_at_1000": 1.0, "precision_at_1": 0.17, "precision_at_3": 0.07, "precision_at_5": 0.048, "precision_at_10": 0.026, "precision_at_20": 0.0145, "precision_at_100": 0.0065, "precision_at_1000": 0.001, "mrr_at_1": 0.17, "mrr_at_3": 0.19, "mrr_at_5": 0.19699999999999998, "mrr_at_10": 0.19922222222222222, "mrr_at_20": 0.20108187134502922, "mrr_at_100": 0.20869401433301007, "mrr_at_1000": 0.20997868463763347, "naucs_at_1_max": 0.517922954216249, "naucs_at_1_std": 0.1114065164377785, "naucs_at_1_diff1": 0.34276389372742455, "naucs_at_3_max": 0.47185139436552825, "naucs_at_3_std": 0.09447796411547679, "naucs_at_3_diff1": 0.2595531062441604, "naucs_at_5_max": 0.41799964054636934, "naucs_at_5_std": 0.11842499400910608, "naucs_at_5_diff1": 0.26475257608435143, "naucs_at_10_max": 0.427987048654274, "naucs_at_10_std": 0.1458038218033593, "naucs_at_10_diff1": 0.30862073949871355, "naucs_at_20_max": 0.4157246267324147, "naucs_at_20_std": 0.1784816559757152, "naucs_at_20_diff1": 0.3228163674573849, "naucs_at_100_max": 0.3920091138416471, "naucs_at_100_std": 0.09514199690780353, "naucs_at_100_diff1": 0.3482463992188136, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |