| {"./data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.622, "ndcg_at_3": 0.67486, "ndcg_at_5": 0.69017, "ndcg_at_10": 0.70946, "ndcg_at_20": 0.72898, "ndcg_at_50": 0.73804, "ndcg_at_100": 0.74397, "map_at_1": 0.622, "map_at_3": 0.662, "map_at_5": 0.6703, "map_at_10": 0.67857, "map_at_20": 0.6841, "map_at_50": 0.68552, "map_at_100": 0.68607, "recall_at_1": 0.622, "recall_at_3": 0.712, "recall_at_5": 0.75, "recall_at_10": 0.808, "recall_at_20": 0.884, "recall_at_50": 0.93, "recall_at_100": 0.966, "precision_at_1": 0.622, "precision_at_3": 0.23733, "precision_at_5": 0.15, "precision_at_10": 0.0808, "precision_at_20": 0.0442, "precision_at_50": 0.0186, "precision_at_100": 0.00966, "mrr_at_1": 0.618, "mrr_at_3": 0.66, "mrr_at_5": 0.6692, "mrr_at_10": 0.6775777777777778, "mrr_at_20": 0.6824860893148819, "mrr_at_50": 0.6840505621420908, "mrr_at_100": 0.6845993176544162, "naucs_at_1_max": 0.46491355378995425, "naucs_at_1_std": -0.06263837723835318, "naucs_at_1_diff1": 0.897580000185275, "naucs_at_3_max": 0.5074962351081752, "naucs_at_3_std": -0.006120744926715115, "naucs_at_3_diff1": 0.8380872858484801, "naucs_at_5_max": 0.5076588173731037, "naucs_at_5_std": 0.00014233385661883033, "naucs_at_5_diff1": 0.818938775510204, "naucs_at_10_max": 0.5770523781854408, "naucs_at_10_std": 0.09178633684734683, "naucs_at_10_diff1": 0.7791151323981074, "naucs_at_20_max": 0.6430128856683821, "naucs_at_20_std": 0.16875963146332645, "naucs_at_20_diff1": 0.7221138397980255, "naucs_at_50_max": 0.6545151393890893, "naucs_at_50_std": 0.26363878884887104, "naucs_at_50_diff1": 0.7213151927437641, "naucs_at_100_max": 0.5877409787444364, "naucs_at_100_std": 0.19736914373592712, "naucs_at_100_diff1": 0.7009666611742652}, "./data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.45676, "ndcg_at_3": 0.52515, "ndcg_at_5": 0.54233, "ndcg_at_10": 0.56915, "ndcg_at_20": 0.5816, "ndcg_at_50": 0.59505, "ndcg_at_100": 0.60258, "map_at_1": 0.45676, "map_at_3": 0.50887, "map_at_5": 0.51829, "map_at_10": 0.52954, "map_at_20": 0.53302, "map_at_50": 0.53529, "map_at_100": 0.53594, "recall_at_1": 0.45676, "recall_at_3": 0.57206, "recall_at_5": 0.61419, "recall_at_10": 0.69623, "recall_at_20": 0.74501, "recall_at_50": 0.81153, "recall_at_100": 0.85809, "precision_at_1": 0.45676, "precision_at_3": 0.19069, "precision_at_5": 0.12284, "precision_at_10": 0.06962, "precision_at_20": 0.03725, "precision_at_50": 0.01623, "precision_at_100": 0.00858, "mrr_at_1": 0.45454545454545453, "mrr_at_3": 0.5066518847006651, "mrr_at_5": 0.5170731707317073, "mrr_at_10": 0.5275815647766867, "mrr_at_20": 0.5311224770899007, "mrr_at_50": 0.5332465170438858, "mrr_at_100": 0.5339250823073732, "naucs_at_1_max": 0.3882948611231304, "naucs_at_1_std": 0.6051243703689616, "naucs_at_1_diff1": 0.8406058479800685, "naucs_at_3_max": 0.3234157964181825, "naucs_at_3_std": 0.6966233741677433, "naucs_at_3_diff1": 0.7729637232110695, "naucs_at_5_max": 0.27276717369737363, "naucs_at_5_std": 0.7006240836207518, "naucs_at_5_diff1": 0.747950791051886, "naucs_at_10_max": 0.16091474340315445, "naucs_at_10_std": 0.7256404433556597, "naucs_at_10_diff1": 0.6981512112733994, "naucs_at_20_max": 0.08633158860631607, "naucs_at_20_std": 0.7747045325493711, "naucs_at_20_diff1": 0.7160198478434746, "naucs_at_50_max": -0.04978574435251967, "naucs_at_50_std": 0.7706903740216889, "naucs_at_50_diff1": 0.6946330790179335, "naucs_at_100_max": -0.1803641306976603, "naucs_at_100_std": 0.8287710376301388, "naucs_at_100_diff1": 0.709766038885269}, "./data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.87, "ndcg_at_3": 0.91786, "ndcg_at_5": 0.92603, "ndcg_at_10": 0.93226, "ndcg_at_20": 0.93226, "ndcg_at_50": 0.93422, "ndcg_at_100": 0.93422, "map_at_1": 0.87, "map_at_3": 0.90667, "map_at_5": 0.91117, "map_at_10": 0.9136, "map_at_20": 0.9136, "map_at_50": 0.9139, "map_at_100": 0.9139, "recall_at_1": 0.87, "recall_at_3": 0.95, "recall_at_5": 0.97, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.87, "precision_at_3": 0.31667, "precision_at_5": 0.194, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.88, "mrr_at_3": 0.9116666666666667, "mrr_at_5": 0.9181666666666666, "mrr_at_10": 0.9195952380952381, "mrr_at_20": 0.9195952380952381, "mrr_at_50": 0.9199077380952382, "mrr_at_100": 0.9199077380952382, "naucs_at_1_max": 0.3063763608087097, "naucs_at_1_std": -0.2671628526994001, "naucs_at_1_diff1": 0.9131674442716432, "naucs_at_3_max": 0.7082166199813253, "naucs_at_3_std": 0.1034547152194234, "naucs_at_3_diff1": 0.8627450980392143, "naucs_at_5_max": 0.8202614379084955, "naucs_at_5_std": 0.0704948646125114, "naucs_at_5_diff1": 0.8638344226579531, "naucs_at_10_max": 0.8692810457516413, "naucs_at_10_std": 0.5541549953314738, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.8692810457516413, "naucs_at_20_std": 0.5541549953314738, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.58809, "ndcg_at_3": 0.70323, "ndcg_at_5": 0.73068, "ndcg_at_10": 0.75075, "ndcg_at_20": 0.75855, "ndcg_at_50": 0.76442, "ndcg_at_100": 0.7668, "map_at_1": 0.58809, "map_at_3": 0.67517, "map_at_5": 0.69036, "map_at_10": 0.69878, "map_at_20": 0.70099, "map_at_50": 0.70196, "map_at_100": 0.70218, "recall_at_1": 0.58809, "recall_at_3": 0.78433, "recall_at_5": 0.85115, "recall_at_10": 0.91252, "recall_at_20": 0.94289, "recall_at_50": 0.97205, "recall_at_100": 0.98663, "precision_at_1": 0.58809, "precision_at_3": 0.26144, "precision_at_5": 0.17023, "precision_at_10": 0.09125, "precision_at_20": 0.04714, "precision_at_50": 0.01944, "precision_at_100": 0.00987, "mrr_at_1": 0.5911300121506683, "mrr_at_3": 0.6764884568651276, "mrr_at_5": 0.6917071688942892, "mrr_at_10": 0.7006767247198595, "mrr_at_20": 0.702628923116812, "mrr_at_50": 0.7035729931078816, "mrr_at_100": 0.7038013299596423, "naucs_at_1_max": 0.16282263440251535, "naucs_at_1_std": -0.2675726048319812, "naucs_at_1_diff1": 0.7855612887331109, "naucs_at_3_max": 0.15869758594775502, "naucs_at_3_std": -0.25252459447099584, "naucs_at_3_diff1": 0.6394956014993703, "naucs_at_5_max": 0.19188935642904395, "naucs_at_5_std": -0.19455295056436547, "naucs_at_5_diff1": 0.6071937017845587, "naucs_at_10_max": 0.2647534611934194, "naucs_at_10_std": -0.054514027748884926, "naucs_at_10_diff1": 0.5405997520135849, "naucs_at_20_max": 0.3763911903387556, "naucs_at_20_std": 0.14592272943721332, "naucs_at_20_diff1": 0.4732245856774079, "naucs_at_50_max": 0.35454897123535783, "naucs_at_50_std": 0.275162782423997, "naucs_at_50_diff1": 0.4532394718684134, "naucs_at_100_max": 0.2978782618422668, "naucs_at_100_std": 0.2997014737148475, "naucs_at_100_diff1": 0.45555904105628975}, "./data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.74494, "ndcg_at_3": 0.79976, "ndcg_at_5": 0.81213, "ndcg_at_10": 0.82846, "ndcg_at_20": 0.83297, "ndcg_at_50": 0.83835, "ndcg_at_100": 0.83933, "map_at_1": 0.74494, "map_at_3": 0.78711, "map_at_5": 0.79389, "map_at_10": 0.8006, "map_at_20": 0.80179, "map_at_50": 0.80272, "map_at_100": 0.8028, "recall_at_1": 0.74494, "recall_at_3": 0.83603, "recall_at_5": 0.8664, "recall_at_10": 0.917, "recall_at_20": 0.93522, "recall_at_50": 0.96154, "recall_at_100": 0.96761, "precision_at_1": 0.74494, "precision_at_3": 0.27868, "precision_at_5": 0.17328, "precision_at_10": 0.0917, "precision_at_20": 0.04676, "precision_at_50": 0.01923, "precision_at_100": 0.00968, "mrr_at_1": 0.742914979757085, "mrr_at_3": 0.7850877192982456, "mrr_at_5": 0.7931848852901484, "mrr_at_10": 0.7990416746995694, "mrr_at_20": 0.8004113500481683, "mrr_at_50": 0.801331531125384, "mrr_at_100": 0.8014399188597515, "naucs_at_1_max": 0.4965959718308746, "naucs_at_1_std": -0.0466933622092674, "naucs_at_1_diff1": 0.8685373039401096, "naucs_at_3_max": 0.5196845357871381, "naucs_at_3_std": 0.014152907602651955, "naucs_at_3_diff1": 0.8084743558786613, "naucs_at_5_max": 0.5925268631205242, "naucs_at_5_std": 0.11651891324846737, "naucs_at_5_diff1": 0.7815885765686577, "naucs_at_10_max": 0.581202610435799, "naucs_at_10_std": 0.23681516182108397, "naucs_at_10_diff1": 0.7349454780601268, "naucs_at_20_max": 0.7922518260743858, "naucs_at_20_std": 0.5495609276681086, "naucs_at_20_diff1": 0.7372706091805555, "naucs_at_50_max": 0.8510476089522382, "naucs_at_50_std": 0.7768090684036366, "naucs_at_50_diff1": 0.6938503074292259, "naucs_at_100_max": 0.8476068082679248, "naucs_at_100_std": 0.8111079242428052, "naucs_at_100_diff1": 0.7003229329465875}, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95393, "ndcg_at_5": 0.9578, "ndcg_at_10": 0.96113, "ndcg_at_20": 0.96113, "ndcg_at_50": 0.96113, "ndcg_at_100": 0.96113, "map_at_1": 0.92, "map_at_3": 0.945, "map_at_5": 0.947, "map_at_10": 0.94843, "map_at_20": 0.94843, "map_at_50": 0.94843, "map_at_100": 0.94843, "recall_at_1": 0.92, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9536666666666666, "mrr_at_10": 0.9550952380952381, "mrr_at_20": 0.9550952380952381, "mrr_at_50": 0.9550952380952381, "mrr_at_100": 0.9550952380952381, "naucs_at_1_max": 0.5140056022408969, "naucs_at_1_std": -0.16888422035480954, "naucs_at_1_diff1": 0.8211367880485536, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.3384687208216551, "naucs_at_3_diff1": 0.9346405228758099, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.12278244631185926, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.56429, "ndcg_at_3": 0.63444, "ndcg_at_5": 0.65026, "ndcg_at_10": 0.66294, "ndcg_at_20": 0.67546, "ndcg_at_50": 0.70544, "ndcg_at_100": 0.71155, "map_at_1": 0.56429, "map_at_3": 0.61786, "map_at_5": 0.62643, "map_at_10": 0.63165, "map_at_20": 0.63503, "map_at_50": 0.63965, "map_at_100": 0.64028, "recall_at_1": 0.56429, "recall_at_3": 0.68214, "recall_at_5": 0.72143, "recall_at_10": 0.76071, "recall_at_20": 0.81071, "recall_at_50": 0.96429, "recall_at_100": 1.0, "precision_at_1": 0.56429, "precision_at_3": 0.22738, "precision_at_5": 0.14429, "precision_at_10": 0.07607, "precision_at_20": 0.04054, "precision_at_50": 0.01929, "precision_at_100": 0.01, "mrr_at_1": 0.5642857142857143, "mrr_at_3": 0.6172619047619048, "mrr_at_5": 0.6267261904761904, "mrr_at_10": 0.6319331065759637, "mrr_at_20": 0.6352391827022797, "mrr_at_50": 0.6399410284324812, "mrr_at_100": 0.6405056533320372, "naucs_at_1_max": 0.250909164994346, "naucs_at_1_std": 0.10353784876791601, "naucs_at_1_diff1": 0.7091792557580451, "naucs_at_3_max": 0.19901092567688966, "naucs_at_3_std": 0.06257940172158412, "naucs_at_3_diff1": 0.6023933615512848, "naucs_at_5_max": 0.16905697042683365, "naucs_at_5_std": 0.025700327070190307, "naucs_at_5_diff1": 0.5975052344915358, "naucs_at_10_max": 0.11986463149253845, "naucs_at_10_std": 0.04068799417636617, "naucs_at_10_diff1": 0.5762841809353437, "naucs_at_20_max": 0.16545256557084664, "naucs_at_20_std": 0.0609616431669272, "naucs_at_20_diff1": 0.511002008899236, "naucs_at_50_max": 0.4613912231559318, "naucs_at_50_std": 0.2222222222222222, "naucs_at_50_diff1": 0.5568160597572381, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.79, "ndcg_at_3": 0.89464, "ndcg_at_5": 0.90238, "ndcg_at_10": 0.90883, "ndcg_at_20": 0.90883, "ndcg_at_50": 0.90883, "ndcg_at_100": 0.90883, "map_at_1": 0.79, "map_at_3": 0.87167, "map_at_5": 0.87567, "map_at_10": 0.87833, "map_at_20": 0.87833, "map_at_50": 0.87833, "map_at_100": 0.87833, "recall_at_1": 0.79, "recall_at_3": 0.96, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.79, "precision_at_3": 0.32, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.85, "mrr_at_3": 0.9016666666666667, "mrr_at_5": 0.9056666666666666, "mrr_at_10": 0.9085833333333333, "mrr_at_20": 0.9085833333333333, "mrr_at_50": 0.9085833333333333, "mrr_at_100": 0.9085833333333333, "naucs_at_1_max": 0.2326688628657996, "naucs_at_1_std": 0.03888234303989203, "naucs_at_1_diff1": 0.9258181643301988, "naucs_at_3_max": 0.7480158730158707, "naucs_at_3_std": 0.6259337068160535, "naucs_at_3_diff1": 0.8231792717086835, "naucs_at_5_max": 0.9346405228758136, "naucs_at_5_std": 0.540149393090577, "naucs_at_5_diff1": 0.9346405228758136, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.48, "ndcg_at_3": 0.59202, "ndcg_at_5": 0.64626, "ndcg_at_10": 0.67623, "ndcg_at_20": 0.68119, "ndcg_at_50": 0.69074, "ndcg_at_100": 0.69411, "map_at_1": 0.48, "map_at_3": 0.565, "map_at_5": 0.5955, "map_at_10": 0.60836, "map_at_20": 0.60966, "map_at_50": 0.61104, "map_at_100": 0.61137, "recall_at_1": 0.48, "recall_at_3": 0.67, "recall_at_5": 0.8, "recall_at_10": 0.89, "recall_at_20": 0.91, "recall_at_50": 0.96, "recall_at_100": 0.98, "precision_at_1": 0.48, "precision_at_3": 0.22333, "precision_at_5": 0.16, "precision_at_10": 0.089, "precision_at_20": 0.0455, "precision_at_50": 0.0192, "precision_at_100": 0.0098, "mrr_at_1": 0.49, "mrr_at_3": 0.5883333333333334, "mrr_at_5": 0.6138333333333333, "mrr_at_10": 0.622702380952381, "mrr_at_20": 0.6240598470157294, "mrr_at_50": 0.6257612271725352, "mrr_at_100": 0.6259278938392019, "naucs_at_1_max": 0.10312797713559874, "naucs_at_1_std": -0.02315549910024338, "naucs_at_1_diff1": 0.6334947602413465, "naucs_at_3_max": -0.0512264901559134, "naucs_at_3_std": -0.15811481099016378, "naucs_at_3_diff1": 0.42873829798039664, "naucs_at_5_max": 0.20339660339660318, "naucs_at_5_std": 0.09702797202797177, "naucs_at_5_diff1": 0.4798701298701295, "naucs_at_10_max": 0.15213469633193094, "naucs_at_10_std": -0.08190877072416533, "naucs_at_10_diff1": 0.5727171205222922, "naucs_at_20_max": 0.265380226164541, "naucs_at_20_std": 0.09165888577653315, "naucs_at_20_diff1": 0.4984956945741262, "naucs_at_50_max": 0.2783613445378158, "naucs_at_50_std": 0.06430905695611724, "naucs_at_50_diff1": 0.1669000933706846, "naucs_at_100_max": -0.22035480859009582, "naucs_at_100_std": -0.22035480859009582, "naucs_at_100_diff1": -0.22035480859009582}, "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95655, "ndcg_at_5": 0.96085, "ndcg_at_10": 0.96085, "ndcg_at_20": 0.96085, "ndcg_at_50": 0.96085, "ndcg_at_100": 0.9625, "map_at_1": 0.92, "map_at_3": 0.94833, "map_at_5": 0.95083, "map_at_10": 0.95083, "map_at_20": 0.95083, "map_at_50": 0.95083, "map_at_100": 0.95098, "recall_at_1": 0.92, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.92, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9516666666666667, "mrr_at_10": 0.9516666666666667, "mrr_at_20": 0.9516666666666667, "mrr_at_50": 0.9516666666666667, "mrr_at_100": 0.9518205128205127, "naucs_at_1_max": 0.5828081232492984, "naucs_at_1_std": 0.2532096171802052, "naucs_at_1_diff1": 0.8768674136321195, "naucs_at_3_max": 0.8611111111111119, "naucs_at_3_std": 0.0793650793650649, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7222222222222276, "naucs_at_5_std": -0.5634920634920767, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": -0.5634920634920767, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": -0.5634920634920767, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 0.7222222222222041, "naucs_at_50_std": -0.5634920634920583, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}} |