{ "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.846, "ndcg_at_3": 0.89433, "ndcg_at_5": 0.8976, "ndcg_at_10": 0.90642, "ndcg_at_20": 0.91194, "ndcg_at_50": 0.91484, "ndcg_at_100": 0.91516, "map_at_1": 0.846, "map_at_3": 0.88267, "map_at_5": 0.88447, "map_at_10": 0.88834, "map_at_20": 0.88984, "map_at_50": 0.89035, "map_at_100": 0.89038, "recall_at_1": 0.846, "recall_at_3": 0.928, "recall_at_5": 0.936, "recall_at_10": 0.962, "recall_at_20": 0.984, "recall_at_50": 0.998, "recall_at_100": 1.0, "precision_at_1": 0.846, "precision_at_3": 0.30933, "precision_at_5": 0.1872, "precision_at_10": 0.0962, "precision_at_20": 0.0492, "precision_at_50": 0.01996, "precision_at_100": 0.01, "mrr_at_1": 0.846, "mrr_at_3": 0.8823333333333332, "mrr_at_5": 0.8850333333333331, "mrr_at_10": 0.8883349206349205, "mrr_at_20": 0.889943956653941, "mrr_at_50": 0.8903646887891893, "mrr_at_100": 0.8903913554558559, "naucs_at_1_max": 0.8081512829311069, "naucs_at_1_std": -0.0772872266066822, "naucs_at_1_diff1": 0.9432244208064874, "naucs_at_3_max": 0.8414124909222956, "naucs_at_3_std": 0.04741155721548222, "naucs_at_3_diff1": 0.9146695715323183, "naucs_at_5_max": 0.8297589869281041, "naucs_at_5_std": 0.02895950046684961, "naucs_at_5_diff1": 0.9121732026143777, "naucs_at_10_max": 0.8905351614329942, "naucs_at_10_std": 0.0927072583419271, "naucs_at_10_diff1": 0.9217406260749892, "naucs_at_20_max": 0.8768674136321192, "naucs_at_20_std": 0.14495798319327652, "naucs_at_20_diff1": 0.8631535947712419, "naucs_at_50_max": 0.86928104575168, "naucs_at_50_std": -1.739962651727529, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.5122, "ndcg_at_3": 0.58642, "ndcg_at_5": 0.60675, "ndcg_at_10": 0.62745, "ndcg_at_20": 0.6406, "ndcg_at_50": 0.65097, "ndcg_at_100": 0.65752, "map_at_1": 0.5122, "map_at_3": 0.56837, "map_at_5": 0.57979, "map_at_10": 0.58828, "map_at_20": 0.59201, "map_at_50": 0.59359, "map_at_100": 0.59419, "recall_at_1": 0.5122, "recall_at_3": 0.63858, "recall_at_5": 0.68736, "recall_at_10": 0.75166, "recall_at_20": 0.80266, "recall_at_50": 0.85588, "recall_at_100": 0.89579, "precision_at_1": 0.5122, "precision_at_3": 0.21286, "precision_at_5": 0.13747, "precision_at_10": 0.07517, "precision_at_20": 0.04013, "precision_at_50": 0.01712, "precision_at_100": 0.00896, "mrr_at_1": 0.5077605321507761, "mrr_at_3": 0.5661492978566152, "mrr_at_5": 0.5760162601626018, "mrr_at_10": 0.5855110336817655, "mrr_at_20": 0.5891507477844685, "mrr_at_50": 0.5906413959072078, "mrr_at_100": 0.5912222399166979, "naucs_at_1_max": 0.1805386230325004, "naucs_at_1_std": 0.2083777775408145, "naucs_at_1_diff1": 0.8899357497181541, "naucs_at_3_max": 0.036250343926384106, "naucs_at_3_std": 0.2729014248258626, "naucs_at_3_diff1": 0.8128872229558094, "naucs_at_5_max": -0.06393649520768908, "naucs_at_5_std": 0.35537819721239466, "naucs_at_5_diff1": 0.7893105740940741, "naucs_at_10_max": -0.2303383632743928, "naucs_at_10_std": 0.4494027929577441, "naucs_at_10_diff1": 0.7907508093846765, "naucs_at_20_max": -0.4436196516643613, "naucs_at_20_std": 0.4884483476927145, "naucs_at_20_diff1": 0.8109543227071737, "naucs_at_50_max": -0.6245429545199258, "naucs_at_50_std": 0.7349758004656695, "naucs_at_50_diff1": 0.8508679773914936, "naucs_at_100_max": -0.8863218597112107, "naucs_at_100_std": 0.9554198503123303, "naucs_at_100_diff1": 0.8643881531474712 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.86842, "ndcg_at_3": 0.91276, "ndcg_at_5": 0.9179, "ndcg_at_10": 0.92323, "ndcg_at_20": 0.92567, "ndcg_at_50": 0.92726, "ndcg_at_100": 0.92757, "map_at_1": 0.86842, "map_at_3": 0.90216, "map_at_5": 0.90509, "map_at_10": 0.90734, "map_at_20": 0.90794, "map_at_50": 0.90819, "map_at_100": 0.90821, "recall_at_1": 0.86842, "recall_at_3": 0.94332, "recall_at_5": 0.95547, "recall_at_10": 0.97166, "recall_at_20": 0.98178, "recall_at_50": 0.98988, "recall_at_100": 0.9919, "precision_at_1": 0.86842, "precision_at_3": 0.31444, "precision_at_5": 0.19109, "precision_at_10": 0.09717, "precision_at_20": 0.04909, "precision_at_50": 0.0198, "precision_at_100": 0.00992, "mrr_at_1": 0.8663967611336032, "mrr_at_3": 0.9018218623481781, "mrr_at_5": 0.90414979757085, "mrr_at_10": 0.9064793393740761, "mrr_at_20": 0.9070689308111907, "mrr_at_50": 0.907317794217949, "mrr_at_100": 0.9073397973864051, "naucs_at_1_max": 0.5849894597264937, "naucs_at_1_std": -0.09180599537618413, "naucs_at_1_diff1": 0.9539022482093868, "naucs_at_3_max": 0.653300720791228, "naucs_at_3_std": -0.21668629871058337, "naucs_at_3_diff1": 0.9135264836941877, "naucs_at_5_max": 0.7483953850859967, "naucs_at_5_std": 0.034054461502585474, "naucs_at_5_diff1": 0.9018156568894024, "naucs_at_10_max": 0.8978323372990794, "naucs_at_10_std": 0.4924193687602959, "naucs_at_10_diff1": 0.9055324021231301, "naucs_at_20_max": 0.9546293120196974, "naucs_at_20_std": 0.7223350051556908, "naucs_at_20_diff1": 0.9419549093045442, "naucs_at_50_max": 0.9183327616354471, "naucs_at_50_std": 0.6841596004695476, "naucs_at_50_diff1": 0.9216391275611305, "naucs_at_100_max": 0.8979159520443043, "naucs_at_100_std": 0.6051995005869166, "naucs_at_100_diff1": 0.9346992729676393 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.90357, "ndcg_at_3": 0.94366, "ndcg_at_5": 0.94781, "ndcg_at_10": 0.95262, "ndcg_at_20": 0.95361, "ndcg_at_50": 0.95431, "ndcg_at_100": 0.95431, "map_at_1": 0.90357, "map_at_3": 0.93512, "map_at_5": 0.93726, "map_at_10": 0.93936, "map_at_20": 0.93968, "map_at_50": 0.93979, "map_at_100": 0.93979, "recall_at_1": 0.90357, "recall_at_3": 0.96786, "recall_at_5": 0.97857, "recall_at_10": 0.99286, "recall_at_20": 0.99643, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.90357, "precision_at_3": 0.32262, "precision_at_5": 0.19571, "precision_at_10": 0.09929, "precision_at_20": 0.04982, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.9333333333333332, "mrr_at_5": 0.9358333333333332, "mrr_at_10": 0.9382879818594103, "mrr_at_20": 0.9382879818594103, "mrr_at_50": 0.938390022675737, "mrr_at_100": 0.938390022675737, "naucs_at_1_max": 0.6358197600027652, "naucs_at_1_std": 0.28033682608845983, "naucs_at_1_diff1": 0.9296261714562363, "naucs_at_3_max": 0.9709513435003594, "naucs_at_3_std": 0.7201473181865297, "naucs_at_3_diff1": 0.9400871459694935, "naucs_at_5_max": 0.9782135076252753, "naucs_at_5_std": 0.7226112667289174, "naucs_at_5_diff1": 0.910130718954251, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 0.7957516339869304, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.72222222222224, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.67922, "ndcg_at_3": 0.77244, "ndcg_at_5": 0.79694, "ndcg_at_10": 0.8121, "ndcg_at_20": 0.8175, "ndcg_at_50": 0.82077, "ndcg_at_100": 0.82237, "map_at_1": 0.67922, "map_at_3": 0.75, "map_at_5": 0.76373, "map_at_10": 0.77012, "map_at_20": 0.7717, "map_at_50": 0.77223, "map_at_100": 0.77238, "recall_at_1": 0.67922, "recall_at_3": 0.83718, "recall_at_5": 0.89611, "recall_at_10": 0.94228, "recall_at_20": 0.96294, "recall_at_50": 0.97934, "recall_at_100": 0.98906, "precision_at_1": 0.67922, "precision_at_3": 0.27906, "precision_at_5": 0.17922, "precision_at_10": 0.09423, "precision_at_20": 0.04815, "precision_at_50": 0.01959, "precision_at_100": 0.00989, "mrr_at_1": 0.6798298906439855, "mrr_at_3": 0.7503037667071691, "mrr_at_5": 0.7639732685297701, "mrr_at_10": 0.7703048756967354, "mrr_at_20": 0.771781703893678, "mrr_at_50": 0.7722855522833878, "mrr_at_100": 0.7724409847415249, "naucs_at_1_max": 0.16344622447696044, "naucs_at_1_std": -0.2550595413752476, "naucs_at_1_diff1": 0.798163282282663, "naucs_at_3_max": 0.25140875244161615, "naucs_at_3_std": -0.19148632297929294, "naucs_at_3_diff1": 0.735908847266564, "naucs_at_5_max": 0.36331170472018576, "naucs_at_5_std": -0.025518611627873482, "naucs_at_5_diff1": 0.7030926615249569, "naucs_at_10_max": 0.3881662571474966, "naucs_at_10_std": 0.19492404683272815, "naucs_at_10_diff1": 0.7210851978418036, "naucs_at_20_max": 0.47757976788208983, "naucs_at_20_std": 0.4380179245317548, "naucs_at_20_diff1": 0.6933989034189479, "naucs_at_50_max": 0.6469748935781172, "naucs_at_50_std": 0.6850989095376581, "naucs_at_50_diff1": 0.7523078962936631, "naucs_at_100_max": 0.6351305556652902, "naucs_at_100_std": 0.667464964759476, "naucs_at_100_diff1": 0.7283558047703392 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.78, "ndcg_at_3": 0.87071, "ndcg_at_5": 0.89137, "ndcg_at_10": 0.89493, "ndcg_at_20": 0.89493, "ndcg_at_50": 0.89493, "ndcg_at_100": 0.89655, "map_at_1": 0.78, "map_at_3": 0.85, "map_at_5": 0.8615, "map_at_10": 0.86317, "map_at_20": 0.86317, "map_at_50": 0.86317, "map_at_100": 0.86331, "recall_at_1": 0.78, "recall_at_3": 0.93, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.78, "precision_at_3": 0.31, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.8, "mrr_at_3": 0.8683333333333334, "mrr_at_5": 0.8758333333333335, "mrr_at_10": 0.8772619047619048, "mrr_at_20": 0.8772619047619048, "mrr_at_50": 0.8772619047619048, "mrr_at_100": 0.8774047619047619, "naucs_at_1_max": 0.17508230166458016, "naucs_at_1_std": -0.3999629062920201, "naucs_at_1_diff1": 0.8873278620114059, "naucs_at_3_max": 0.32412965186074477, "naucs_at_3_std": -0.44864612511671287, "naucs_at_3_diff1": 0.8382686407896491, "naucs_at_5_max": 0.6136788048552745, "naucs_at_5_std": 0.35807656395892185, "naucs_at_5_diff1": 0.7117180205415541, "naucs_at_10_max": 0.35807656395891135, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 0.5541549953314738, "naucs_at_20_max": 0.35807656395891135, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.5541549953314738, "naucs_at_50_max": 0.35807656395892007, "naucs_at_50_std": 0.35807656395892007, "naucs_at_50_diff1": 0.554154995331464, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.99, "ndcg_at_3": 0.99631, "ndcg_at_5": 0.99631, "ndcg_at_10": 0.99631, "ndcg_at_20": 0.99631, "ndcg_at_50": 0.99631, "ndcg_at_100": 0.99631, "map_at_1": 0.99, "map_at_3": 0.995, "map_at_5": 0.995, "map_at_10": 0.995, "map_at_20": 0.995, "map_at_50": 0.995, "map_at_100": 0.995, "recall_at_1": 0.99, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.99, "mrr_at_3": 0.995, "mrr_at_5": 0.995, "mrr_at_10": 0.995, "mrr_at_20": 0.995, "mrr_at_50": 0.995, "mrr_at_100": 0.995, "naucs_at_1_max": 0.7222222222222201, "naucs_at_1_std": 0.7222222222222201, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.95, "ndcg_at_3": 0.96131, "ndcg_at_5": 0.96562, "ndcg_at_10": 0.97184, "ndcg_at_20": 0.97184, "ndcg_at_50": 0.97184, "ndcg_at_100": 0.97184, "map_at_1": 0.95, "map_at_3": 0.95833, "map_at_5": 0.96083, "map_at_10": 0.96326, "map_at_20": 0.96326, "map_at_50": 0.96326, "map_at_100": 0.96326, "recall_at_1": 0.95, "recall_at_3": 0.97, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.32333, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.9583333333333333, "mrr_at_5": 0.9608333333333333, "mrr_at_10": 0.9633730158730158, "mrr_at_20": 0.9633730158730158, "mrr_at_50": 0.9633730158730158, "mrr_at_100": 0.9633730158730158, "naucs_at_1_max": 0.5883286647992536, "naucs_at_1_std": -0.408309990662932, "naucs_at_1_diff1": 0.9738562091503253, "naucs_at_3_max": 0.5714285714285686, "naucs_at_3_std": -1.040616246498596, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.4960317460317504, "naucs_at_5_std": -1.7399626517273414, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.89, "ndcg_at_3": 0.95178, "ndcg_at_5": 0.95609, "ndcg_at_10": 0.95609, "ndcg_at_20": 0.95609, "ndcg_at_50": 0.95609, "ndcg_at_100": 0.95609, "map_at_1": 0.89, "map_at_3": 0.93833, "map_at_5": 0.94083, "map_at_10": 0.94083, "map_at_20": 0.94083, "map_at_50": 0.94083, "map_at_100": 0.94083, "recall_at_1": 0.89, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.89, "mrr_at_3": 0.94, "mrr_at_5": 0.9425, "mrr_at_10": 0.9425, "mrr_at_20": 0.9425, "mrr_at_50": 0.9425, "mrr_at_100": 0.9425, "naucs_at_1_max": 0.4527531998969162, "naucs_at_1_std": 0.3915041663087361, "naucs_at_1_diff1": 0.8127308650459574, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.5541549953314585, "naucs_at_3_diff1": 0.8692810457516356, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.97, "ndcg_at_3": 0.98262, "ndcg_at_5": 0.98693, "ndcg_at_10": 0.98693, "ndcg_at_20": 0.98693, "ndcg_at_50": 0.98693, "ndcg_at_100": 0.98693, "map_at_1": 0.97, "map_at_3": 0.98, "map_at_5": 0.9825, "map_at_10": 0.9825, "map_at_20": 0.9825, "map_at_50": 0.9825, "map_at_100": 0.9825, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.97, "mrr_at_3": 0.98, "mrr_at_5": 0.9825, "mrr_at_10": 0.9825, "mrr_at_20": 0.9825, "mrr_at_50": 0.9825, "mrr_at_100": 0.9825, "naucs_at_1_max": 0.5448179271708694, "naucs_at_1_std": 0.044817927170871553, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 0.7222222222222157, "naucs_at_3_std": -0.5634920634921204, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } }