add gemini-flash-2.0-exp

constants.py

@@ -118,6 +118,7 @@ MODEL_NAME_MAP = {
     "InternVL2_5_8B": "InternVL2.5-8B",
     "Grok-2-vision-1212": "Grok-2-vision-1212",
     "Gemini-2.0-thinking": "Gemini-2.0-flash-thinking",
+    "Gemini-Flash-2.0-exp": "Gemini-Flash-2.0-exp",
     "Gemini-exp-1206": "Gemini-exp-1206",
 }
 
@@ -209,15 +210,16 @@ MODEL_URLS = {
     "Grok-2-vision-1212": "https://x.ai/blog/grok-1212",
     "Gemini-2.0-thinking": "://ai.google.dev/gemini-api/docs/thinking-mode",
     "Gemini-exp-1206": "https://blog.google/feed/gemini-exp-1206/",
+    "Gemini-Flash-2.0-exp": "https://blog.google/technology/google-deepmind/google-gemini-ai-update-december-2024/#gemini-2-0-flash",
 }
 
 # Define the base MODEL_GROUPS structure
 BASE_MODEL_GROUPS = {
     "All": list(MODEL_NAME_MAP.keys()),
-    "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', 'Molmo_72B', 'InternVL2_5_78B', 'Grok-2-vision-1212', "Gemini-2.0-thinking", "Gemini-exp-1206"],
-    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B"],
-    "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Grok-2-vision-1212', "Gemini-2.0-thinking", "Gemini-exp-1206"],
-    "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini'],
+    "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', 'Molmo_72B', 'InternVL2_5_78B', 'Grok-2-vision-1212', "Gemini-exp-1206"],
+    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Gemini-2.0-thinking", "Gemini-Flash-2.0-exp"],
+    "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Grok-2-vision-1212', "Gemini-exp-1206"],
+    "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', "Gemini-Flash-2.0-exp", "Gemini-2.0-thinking"],
     "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', "Molmo_72B", "InternVL2_5_78B"],
     "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B"]
 }

static/eval_results/Default/Gemini-Flash-2.0-exp/summary_and_keyword_stats.json

@@ -0,0 +1,249 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6531,
+            "macro_mean_score": 0.5407469682493938
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1158,
+            "macro_mean_score": 0.632968160070652
+        },
+        "overall_score": 0.552617022642229
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4745,
+                "tasks": [],
+                "average_score": 0.5782979256943778
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2503,
+                "tasks": [],
+                "average_score": 0.6135285008667205
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 853,
+                "tasks": [],
+                "average_score": 0.6466029104690044
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.574739062334026
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1385,
+                "tasks": [],
+                "average_score": 0.5672870648345623
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2434,
+                "tasks": [],
+                "average_score": 0.41187777635464107
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6641203007518797
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2232,
+                "tasks": [],
+                "average_score": 0.59555536867041
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1908,
+                "tasks": [],
+                "average_score": 0.47799127604909475
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 576,
+                "tasks": [],
+                "average_score": 0.258060016054743
+            }
+        },
+        "input_format": {
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2243,
+                "tasks": [],
+                "average_score": 0.5805088447248222
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 540,
+                "tasks": [],
+                "average_score": 0.5716301924585377
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5056938373064443
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1717,
+                "tasks": [],
+                "average_score": 0.5282795652139135
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.49897892967383023
+            },
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1511,
+                "tasks": [],
+                "average_score": 0.6095454500115458
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.4601450277175127
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1511,
+                "tasks": [],
+                "average_score": 0.5296197886902184
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1449,
+                "tasks": [],
+                "average_score": 0.6089096459304206
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1713,
+                "tasks": [],
+                "average_score": 0.4969695210701893
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.5234237145984363
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.5912336308948948
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1274,
+                "tasks": [],
+                "average_score": 0.5769496085438593
+            }
+        },
+        "input_num": {
+            "1-image": {
+                "count": 315,
+                "num_samples": 5215,
+                "tasks": [],
+                "average_score": 0.5772282223913567
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5056938373064443
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.4884965564296879
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.6019278856911068
+            },
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.4508408919123204
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.4851820865640744
+            }
+        },
+        "app": {
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1602,
+                "tasks": [],
+                "average_score": 0.625268184642034
+            },
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1119,
+                "tasks": [],
+                "average_score": 0.6396914731086236
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2310,
+                "tasks": [],
+                "average_score": 0.5719180669097592
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.4948051263516159
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.5678501530733836
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1237,
+                "tasks": [],
+                "average_score": 0.3603705820786078
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.6566188392926626
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.49656912770604317
+            }
+        }
+    }
+}

static/eval_results/Default/Gemini-Flash-2.0-exp/task_results.json

@@ -0,0 +1,3537 @@
+[
+    {
+        "name": "ascii_art_30",
+        "score": 0.21428571428571427,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "humor_explanation",
+        "score": 0.8533333333333335,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "funqa_unexpected_action_humor_video",
+        "score": 0.3733333333333333,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "science_figure_explanation",
+        "score": 0.872413793103448,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "video_qa",
+        "score": 0.8571428571428573,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vibe_eval_phrase",
+        "score": 0.7571428571428573,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "traffic_accident_analysis",
+        "score": 0.6714285714285714,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "figurative_speech_explanation",
+        "score": 0.8344827586206895,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "video_detail_description",
+        "score": 0.5210526315789474,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "table2latex_complex",
+        "score": 0.8111111111111111,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "unusual_images",
+        "score": 0.889655172413793,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "funqa_unexpected_action_creative_video",
+        "score": 0.3866666666666667,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "art_explanation",
+        "score": 0.4655172413793103,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "ocr_open_ended_qa",
+        "score": 0.7965517241379308,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "video_short_title",
+        "score": 0.6928571428571428,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bar_chart_interpretation",
+        "score": 0.6655172413793102,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "activitynetqa",
+        "score": 0.4473684210526316,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "scibench_w_solution_open_ended",
+        "score": 0.618,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 25
+    },
+    {
+        "name": "doc_vqa",
+        "score": 0.7562500000000001,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "video2notes",
+        "score": 0.7000000000000001,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Chat_Hard",
+        "score": 0.4115384615384616,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 26
+    },
+    {
+        "name": "image_humor_understanding",
+        "score": 0.882758620689655,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "video_summary",
+        "score": 0.692857142857143,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "defeasible_reasoning",
+        "score": 0.8620689655172413,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "funny_image_title",
+        "score": 0.6499999999999998,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_content_follow_up",
+        "score": 0.7285714285714286,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "nextqa_oe",
+        "score": 0.4157894736842105,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "tweets_captioning",
+        "score": 0.5499999999999999,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "wikihow_complex_task_completion",
+        "score": 0.7555555555555555,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "graph_interpretation",
+        "score": 0.8413793103448274,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "generated_video_artifacts",
+        "score": 0.3625,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "meme_explain",
+        "score": 0.8857142857142858,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "funqa_unexpected_action_magic_video",
+        "score": 0.6466666666666667,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "guess_image_generation_prompt",
+        "score": 0.8263157894736844,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "visualization_with_code",
+        "score": 0.6714285714285715,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "iq_test_open_ended",
+        "score": 0.6689655172413791,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "electrocardiogram",
+        "score": 0.3857142857142857,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "image_captioning_with_additional_requirements",
+        "score": 0.9285714285714287,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "docci_image_description_long",
+        "score": 0.7928571428571428,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "paper_review_writing",
+        "score": 0.5866666666666666,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "sceneqa_scene_transition_video",
+        "score": 0.34285714285714286,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Chat_Easy",
+        "score": 0.5769230769230769,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 26
+    },
+    {
+        "name": "bridge_strategies_advanced",
+        "score": 0.15714285714285717,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bridge_strategies_worldclass",
+        "score": 0.08571428571428572,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bridge_strategies_expert",
+        "score": 0.19999999999999998,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_manual_explanation_scooter_Arabic",
+        "score": 0.4928571428571428,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_manual_explanation_scooter_Russian",
+        "score": 0.46428571428571425,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_manual_explanation_scooter_French",
+        "score": 0.4785714285714286,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_manual_explanation_scooter_Spanish",
+        "score": 0.5142857142857143,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_manual_explanation_scooter_Chinese",
+        "score": 0.5285714285714286,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Spanish",
+        "score": 0.7428571428571429,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Japanese",
+        "score": 0.6857142857142857,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_French",
+        "score": 0.7357142857142858,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Arabic",
+        "score": 0.7000000000000001,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Russian",
+        "score": 0.6857142857142857,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_English",
+        "score": 0.7285714285714285,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "table_understanding_fetaqa",
+        "score": 0.5928571428571429,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "red_teaming_celebrity",
+        "score": 0.8550000000000001,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "red_teaming_captcha",
+        "score": 0.13157894736842107,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "red_teaming_jailbreak",
+        "score": 0.5850000000000001,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "red_teaming_visualmisleading",
+        "score": 0.8789473684210528,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "red_teaming_visual_order_A",
+        "score": 0.8947368421052634,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "red_teaming_racial",
+        "score": 0.7600000000000001,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "red_teaming_visual_order_B",
+        "score": 0.9,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "red_teaming_politics",
+        "score": 0.695,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "brand_logo_recognition_and_elaboration",
+        "score": 0.88,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 25
+    },
+    {
+        "name": "exchange_rate_estimate_plot",
+        "score": 0.9733571428571428,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "places365_similar_scene_retrieval",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "math_parity",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "av_human_multiview_counting",
+        "score": 0.06666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "traffic_future_prediction_from_line_plot",
+        "score": 0.6567894736842107,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "graph_chordless_cycle",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_load_type_prediction_from_plot",
+        "score": 0.4761904761904761,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "youtube_video_info_parsing",
+        "score": 0.7976190476190476,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "super_clevr_scene_understanding",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "figureqa",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_programming_test_advanced",
+        "score": 0.1111111111111111,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 18
+    },
+    {
+        "name": "face_keypoint_detection",
+        "score": 0.5787325110618333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "widerface_face_count_and_event_classification",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "average_humidity_estimate_plot",
+        "score": 0.8413333333333332,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "code_output_result",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "clevrer_video_moving_object_count",
+        "score": 0.3333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 21
+    },
+    {
+        "name": "weather_info_parsing",
+        "score": 0.896825396825397,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "egocentric_analysis_single_image",
+        "score": 0.4444444444444444,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "logo2k_same_type_logo_retrieval",
+        "score": 0.9642857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_hamiltonian_cycle",
+        "score": 0.45357142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "waybill_number_sequence_extraction",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_maxflow",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "clevrer_object_existence_video",
+        "score": 0.5625,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "TV_show_info_parsing",
+        "score": 0.8015873015873015,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "sta_action_localization_video",
+        "score": 0.3125,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "visual_dialog_image_guessing",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "insect_order_classification",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "electricity_plot_future_prediction",
+        "score": 0.8230157894736841,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "perception_test_video_character_order",
+        "score": 0.875,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "chemistry_exams_v",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vlnqa_egocentric_navigation_video",
+        "score": 0.4375,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "finance_table_understanding",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "funsd_document_qa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "star_object_interaction_video",
+        "score": 0.625,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "video_to_camera_trajectory_retrieval",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vibe_eval_open",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "question_solution_solving",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_theory",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_programming_test_hard",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_analytic",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "3d_fragments_understanding",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_length",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "algebra",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_puzzle_single_step",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "action_sequence_understanding",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "emotion_recognition",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_winner_identification",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "perception_test_object_shuffle_video",
+        "score": 0.25,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "physical_property_reasoning",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "humor_understand_caption_match",
+        "score": 0.7333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "coco_object_detection_by_query_property",
+        "score": 0.6698026360094582,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "cam_traj_to_video_selection",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multilingual_game_info_parsing",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mnist_pattern",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "cheapest_flight_identification",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "newspaper_page_parse_and_count",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "clevrer_moving_direction_video",
+        "score": 0.1875,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "dvqa",
+        "score": 0.7894736842105263,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "science_basic_physics",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "electricity_future_prediction_from_table",
+        "score": 0.737017543859649,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "physics_exams_v",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "license_plate_recognition",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "snli_ve_visual_entailment",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "game_info_retrieval",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "places365_scene_type_classification",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_info_retrieval",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "3d_indoor_scene_text_bbox_selection",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_error_line_identification",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_descriptive",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "top_rated_hotel_identification",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "science_molecule_chemistry",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "face_identity_matching",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "game_info_parsing",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_info_parsing",
+        "score": 0.6071428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "deciphering_oracle_bone",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_content_reasoning",
+        "score": 0.2222222222222222,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "multilingual_movie_info_parsing",
+        "score": 0.7346938775510203,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "iconqa_count_and_reasoning",
+        "score": 0.631578947368421,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "graph_connectivity",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "graph_shortest_path_planar",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "famous_building_recognition",
+        "score": 0.9375,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "muma_theory_of_mind_belief_of_goal",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "signboard_identification",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_transformation",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "image_style_recognition",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "math_convexity_value_estimation",
+        "score": 0.5207276876255985,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "av_vehicle_multiview_counting",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "long_string_letter_recognition",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "3d_indoor_scene_text_bbox_prediction",
+        "score": 0.19380819004670155,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "movie_info_parsing",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "av_view_identification",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "handwritten_math_expression_extraction",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_solid",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "arc_agi",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "animal_pose_estimation",
+        "score": 0.3668898610562918,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "single_person_pose_estimation",
+        "score": 0.298785583093911,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "next_action_prediction",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "human_relationship_reasoning",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_shortest_path_kamada_kawai",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_area",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "coco_person_detection",
+        "score": 0.6026048935518167,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chart_vqa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "IAM_line_ocr_and_locate",
+        "score": 0.6713087080689123,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "hotel_booking_confirmation_parsing",
+        "score": 0.7071428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "clevrer_video_moving_object_property_recognition",
+        "score": 0.5625,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "ili_ratio_future_prediction",
+        "score": 0.16085714285714284,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "landmark_check_two_images",
+        "score": 0.7777777777777779,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "graph_hamiltonian_path",
+        "score": 0.324404761904762,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vizwiz_quality_accessment_for_blind",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "nlvr2_two_image_compare_qa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "electricity_load_estimate_plot",
+        "score": 0.7926428571428572,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "weather_info_retrieval",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "tqa_textbook_qa",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "stock_info_parsing",
+        "score": 0.9663865546218489,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "math_exams_v",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "coco_ood_global_image_retrieval_by_query_property",
+        "score": 0.7928571428571428,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "quizlet_question_solving",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "newspaper_ocr_in_query_box",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "clevr_arithmetic",
+        "score": 0.5789473684210527,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "stock_info_retrieval",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mvsa_sentiment_classification",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_programming_extremely_hard",
+        "score": 0.0625,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "av_multicamera_tracking_predict_bbox",
+        "score": 0.04470906681625265,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "egocentric_spatial_reasoning",
+        "score": 0.5555555555555556,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "stock_price_future_prediction",
+        "score": 0.8637857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "Ad_count_detection",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "recover_masked_word_in_figure",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "polygon_interior_angles",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "web_action_grounding",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "latex_complex_formula_convertion",
+        "score": 0.29411764705882354,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "transit_map_intersection_points",
+        "score": 0.3898809523809524,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "paper_review_acceptance",
+        "score": 0.4666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "arxiv_vqa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "perception_test_video_action_count",
+        "score": 0.3125,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "code_match_problem",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vln_hindi_next_step",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "medical_image_artifacts_indentification",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "song_title_identification_from_lyrics",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_abdomen_endscopy_organ_recognition",
+        "score": 0.3154761904761904,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "move_pos_to_pos_hanoi_4_pole",
+        "score": 0.03571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_camera_motion_description",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "actor_recognition_in_Movie",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bongard_problem",
+        "score": 0.42105263157894735,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "ascii_art_understanding",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "calendar_schedule_suggestion",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_reasoning_overlapped_circle",
+        "score": 0.75,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_barman",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "planning_screenshot_floortile",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "nextqa_mc",
+        "score": 0.7894736842105263,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "graph_isomorphism",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "code_programming_test_easy",
+        "score": 0.375,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 24
+    },
+    {
+        "name": "biology_exams_v",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "long_string_number_recognition",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "kvqa_knowledge_aware_qa",
+        "score": 0.42105263157894735,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "video_grounding_temporal",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "math_breakpoint",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "ancient_map_understanding",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "landmark_recognition_and_qa",
+        "score": 0.7111111111111111,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "code_execution",
+        "score": 0.8125,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "music_sheet_format_QA",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_solution_compare",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "annoying_word_search",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "interpret_force_perspective_illusion",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "muma_theory_of_mind_social_goal",
+        "score": 0.4666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "healthcare_info_judgement",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "photo_sharing_image_retrieval",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiview_reasoning_camera_moving",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_plot_position_relationship",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "map_diagram_qa",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "movie_info_retrieval",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vln_identify_location",
+        "score": 0.2121212121212121,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "pmc_vqa_medical_image_qa",
+        "score": 0.5789473684210527,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "medical_polyp_segmentation_single_object_rater",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "logical_reasoning_2D_views_of_3D_shapes",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_blood_vessels_recognition",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_abdomen_MRI_organ_recognition",
+        "score": 0.3988095238095238,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "relative_depth_of_different_points",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "google_streetview_circle_sorting",
+        "score": 0.06666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "location_vqa",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "topological_sort",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mindmap_elements_parsing",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_add_tag",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "painting_QA",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vln_identify_robot",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "planning_visual_barman",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "TRANCE_physics_reasoning_view",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_translation_hard",
+        "score": 0.05952380952380952,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "scibench_fundamental_wo_solution",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 49
+    },
+    {
+        "name": "2d_image_jigsaw_puzzle_easy",
+        "score": 0.1976190476190476,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_reasoning_nested_squares",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "font_recognition",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "rocks_samples_compare",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mensa_iq_test",
+        "score": 0.4372549019607843,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "dish_ingredient_match",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "flowchart_code_generation",
+        "score": 0.5555555555555556,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "geometry_reasoning_count_line_intersections",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "functionality_matching_in_different_objects",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "worldle",
+        "score": 0.5549424986515583,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "stackoverflow_debug_QA",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "logical_reasoning_find_odd_one_out",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "circuit_diagram_understanding",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "web_action_prediction",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_retrieval_given_surgeon_activity",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "google_streetview_line_sorting",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "recipe_image_ordering",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_sheet_sentiment",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "TRANCE_physics_reasoning_basic",
+        "score": 0.7058823529411765,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "signage_navigation",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "go_capture_stone",
+        "score": 0.13333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "google_streetview_direction_understanding",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_translation_advanced",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_retrieval",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "comic_page_ordering",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "movie_retrieval_by_actor",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "webpage_code_understanding",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "counting_multi_image",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_eval_visual_pref",
+        "score": 0.5625,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "visual_correspondance_in_two_images",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "forensic_detection_of_different_images",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "sign_language",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "monthly_weather_days_count",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_counting_lymphocytes",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "weather_map_climate_type_temperature_parsing",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "knowledge_sign_recognition",
+        "score": 0.4444444444444444,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "top_video_creator_identification",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_visualization_output_understanding",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 10
+    },
+    {
+        "name": "TRANCE_physics_reasoning_event",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "rebus",
+        "score": 0.43478260869565216,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 23
+    },
+    {
+        "name": "vln_tegulu_next_step",
+        "score": 0.13333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "video_grounding_spatial",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ishihara_test",
+        "score": 0.19285714285714287,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_sheet_name",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "paper_vqa",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vln_english_next_step",
+        "score": 0.13333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "google_streetview_circle_reasoning",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "product_ocr_qa",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multilingual_news_qa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_reasoning_circled_letter",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_sheet_author",
+        "score": 0.125,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "GUI_Act_Web_Single",
+        "score": 0.026658939437844104,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_visual_blocksworld",
+        "score": 0.13333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "game_platform_support_identification",
+        "score": 0.9642857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Act_Mobile_swipe",
+        "score": 0.5319532782862981,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 13
+    },
+    {
+        "name": "mahjong",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "booking_web_recommendation",
+        "score": 0.7117630385487528,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_eval_dynamic_pref",
+        "score": 0.9375,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "extract_webpage_headline",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_action_recognition",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_storage",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "scibench_calculus_wo_solution",
+        "score": 0.4489795918367347,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 49
+    },
+    {
+        "name": "knowledge_graph_understanding",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "media_homepage_profile",
+        "score": 0.21281397174254318,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "image_translation_en2cn",
+        "score": 0.4747283144053553,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "google_streetview_line_reasoning",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "realworld_qa_en2cn",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_translation_easy",
+        "score": 0.5952380952380951,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "soccer_offside",
+        "score": 0.5555555555555556,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "planning_visual_floortile",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "planning_visual_storage",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "video_segments_reordering",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_parasite_detection",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_intent_recognition",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_reasoning_grid",
+        "score": 0.75,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Act_Web_Multi",
+        "score": 0.23629435627595965,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chinese_idiom_recognition",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "relative_reflectance_of_different_regions",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "number_comparison",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "entertainment_web_game_style",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "media_recommend_solutions_stackoverflow",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "orchestra_score_recognition",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_visual_termes",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "video_eval_factual_pref",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_blocksworld",
+        "score": 0.3333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "icon_arithmetic_puzzle",
+        "score": 0.6785714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_visual_grippers",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "planning_screenshot_grippers",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "character_recognition_in_TV_shows",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "highest_discount_game_price_identification",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "remaining_playback_time_calculation",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_cell_recognition",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "pokemon_3D_recognition",
+        "score": 0.7333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "chess_find_legal_moves",
+        "score": 0.20779353910574844,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "rocks_samples_identify",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "paper_review_rating",
+        "score": 0.7641625001389684,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "distinguish_ai_generated_image",
+        "score": 0.8421052631578947,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "autonomous_driving_scene_analysis",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "code_translation_Python",
+        "score": 0.5416666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "counting_single_image",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMMU_pro_exam_screenshot",
+        "score": 0.40404040404040403,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 99
+    },
+    {
+        "name": "GUI_Act_Mobile_tap",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "road_map_find_highway_between_two_place",
+        "score": 0.8235294117647058,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "waldo",
+        "score": 0.001960708071909715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 18
+    },
+    {
+        "name": "clevrer_physics",
+        "score": 0.45,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 20
+    },
+    {
+        "name": "chess_sygyzy_endgames",
+        "score": 0.15739022881880024,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "llavaguard",
+        "score": 0.5357142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "photoshop_operation",
+        "score": 0.29523809523809524,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMMU_physics_chemistry_selected",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_multi_organ_segmentation_rater",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "cultural_vqa",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "medical_content_based_retrieval_radiology",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "logical_reasoning_fit_pattern",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_tyreworld",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "tv_show_retrieval_by_character",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_sheet_note_count",
+        "score": 0.058823529411764705,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "semantic_matching_of_two_images",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_keywords_based_retrieval_non_radiology",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "booking_web_rating",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_termes",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "geographic_remote_sensing_land_cover",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "logical_reasoning_2d_folding",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "hashtag_recommendation",
+        "score": 0.9166666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_plane_segmentation",
+        "score": 0.5111111111111112,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "multiple_states_identify_europe",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_americas",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_surface_normal_estimation",
+        "score": 0.6904761904761905,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_distance",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_openable_part_segmentation",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_count",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_depth",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_3d_assembled_quality_understanding",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_relation",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_semantic_segmentation",
+        "score": 0.5208333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "symbolic_graphics_programs_computer_aided_design",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_depth_estimation",
+        "score": 0.619047619047619,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_novel_view_synthesis",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "symbolic_graphics_programs_scalable_vector_graphics",
+        "score": 0.1111111111111111,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 18
+    },
+    {
+        "name": "table_understanding_complex_question_answering",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "visual_prediction_rater_panoptic_segmentation",
+        "score": 0.5476190476190476,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "table_understanding_fact_verification",
+        "score": 0.9047619047619048,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "panel_images_multi_question",
+        "score": 0.8095238095238095,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_asia",
+        "score": 0.9000000000000001,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "panel_images_single_question",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_africa",
+        "score": 0.8142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMSoc_Misinformation_GossipCop",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMSoc_HatefulMemes",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "poetry_petrarchian_sonnet_optional_meter",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "MMSoc_Memotion",
+        "score": 0.6117647058823531,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "poetry_haiku",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "MMSoc_Misinformation_PolitiFact",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "poetry_shakespearean_sonnet",
+        "score": 0.06666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "poetry_acrostic_alliteration",
+        "score": 0.7333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "screenshot_lighteval_math",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "poetry_acrostic",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "poetry_limerick",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "screenshot_theoremqa",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "poetry_custom_rhyming_scheme",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "text_entity_replace",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "background_change",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "face_attribute_edit",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "face_swap",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "text_style",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "number_puzzle_sudoku",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "out_of_context",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "clip_stable_diffusion_generate",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "veracity",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "counterfactual_arithmetic",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "maze_2d_8x8",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "shape_composition_shapes",
+        "score": 0.47176870748299315,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "shape_composition_colours",
+        "score": 0.4498299319727891,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_aesthetics",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_unmask",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "number_puzzle_kakuro_5x5",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "autorater_semantics",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_iphone_settings",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_amazon",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_motion_guided_editing",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_tiktok",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_artifact",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_ppt",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_mask",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_alipay",
+        "score": 0.6470588235294118,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "autorater_subject",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_leetcode",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_excel",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_zoom",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "autorater_control",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_youtube",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_twitter",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_word",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_3d_model_texturing",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_instagram",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_interactive_operations_notes",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_artifact_reason",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "chess_puzzles_crushing",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_amazon",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_puzzles_checkmate",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_instagram",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_puzzles_equality",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "app_layout_understanding_zoom",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "app_layout_understanding_notes",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_word",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_twitter",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_iphone_settings",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_youtube",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_leetcode",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_ppt",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_tiktok",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_alipay",
+        "score": 0.8235294117647058,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "app_layout_understanding_excel",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_employer_plain",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_article_journal",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_experience_plain",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ball_cup_swap_3",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_markdown",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_math_text_latex",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_latex",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_school_plain",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_article_authors",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_html",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_skill_plain",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_csv",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "crossword_mini_5x5",
+        "score": 0.6714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_math_equation",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "contain_repeat_length",
+        "score": 0.4666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "multi_contain_position_only",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "contain_position_images",
+        "score": 0.3333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "xor_images",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "contain_position_length",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "multi_contain_repeat",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "multi_contain_repeat_position_only_length",
+        "score": 0.06666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "pictionary_skribbl_io",
+        "score": 0.3,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 20
+    },
+    {
+        "name": "pictionary_doodle_guess",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "contain_contain_images",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "pictionary_genai_output_chinese",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "pictionary_cartoon_drawing_guess",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "contain_length",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "pictionary_chinese_food_img2en",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "contain_contain_length",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "reward_models_t2i_reward",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "reward_models_i2t_reward",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "memorization_chinese_celebrity",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "memorization_papers",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "memorization_famous_treaty",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "memorization_indian_celebrity",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "research_website_parsing_blogpost",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "research_website_parsing_publication",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "research_website_parsing_homepage",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_motion_matching_real_3D",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "video_motion_matching_3D_real",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    }
+]

static/eval_results/SI/Gemini-Flash-2.0-exp/summary_and_keyword_stats.json

@@ -0,0 +1,213 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 273,
+            "num_eval_samples": 4108,
+            "macro_mean_score": 0.564920890288835
+        },
+        "open": {
+            "num_eval_tasks": 42,
+            "num_eval_samples": 808,
+            "macro_mean_score": 0.6572258810577452
+        },
+        "overall_score": 0.5772282223913563
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 172,
+                "num_samples": 2704,
+                "tasks": [],
+                "average_score": 0.596725933625478
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 46,
+                "num_samples": 896,
+                "tasks": [],
+                "average_score": 0.5808105220673139
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 91,
+                "num_samples": 1628,
+                "tasks": [],
+                "average_score": 0.5140802058578953
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 78,
+                "num_samples": 1270,
+                "tasks": [],
+                "average_score": 0.45162462384154295
+            },
+            "Text Recognition (OCR)": {
+                "count": 101,
+                "num_samples": 1680,
+                "tasks": [],
+                "average_score": 0.6001986640309387
+            },
+            "Scene and Event Understanding": {
+                "count": 60,
+                "num_samples": 1004,
+                "tasks": [],
+                "average_score": 0.6719749185963775
+            },
+            "Language Understanding and Generation": {
+                "count": 102,
+                "num_samples": 1707,
+                "tasks": [],
+                "average_score": 0.6431550536063879
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 38,
+                "num_samples": 652,
+                "tasks": [],
+                "average_score": 0.6799316098253394
+            },
+            "Planning and Decision Making": {
+                "count": 23,
+                "num_samples": 355,
+                "tasks": [],
+                "average_score": 0.20199150554951897
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 10,
+                "num_samples": 170,
+                "tasks": [],
+                "average_score": 0.7211804511278196
+            }
+        },
+        "input_format": {
+            "Photographs": {
+                "count": 83,
+                "num_samples": 1310,
+                "tasks": [],
+                "average_score": 0.6249728230779608
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 88,
+                "num_samples": 1523,
+                "tasks": [],
+                "average_score": 0.5662031698153198
+            },
+            "User Interface Screenshots": {
+                "count": 67,
+                "num_samples": 1117,
+                "tasks": [],
+                "average_score": 0.5760042013946798
+            },
+            "Text-Based Images and Documents": {
+                "count": 53,
+                "num_samples": 847,
+                "tasks": [],
+                "average_score": 0.5055178839706232
+            },
+            "Artistic and Creative Content": {
+                "count": 22,
+                "num_samples": 388,
+                "tasks": [],
+                "average_score": 0.6451175637719202
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 2,
+                "num_samples": 30,
+                "tasks": [],
+                "average_score": 0.27547552359477934
+            }
+        },
+        "output_format": {
+            "structured_output": {
+                "count": 72,
+                "num_samples": 1120,
+                "tasks": [],
+                "average_score": 0.5687115330249284
+            },
+            "numerical_data": {
+                "count": 39,
+                "num_samples": 694,
+                "tasks": [],
+                "average_score": 0.5489576369705605
+            },
+            "multiple_choice": {
+                "count": 33,
+                "num_samples": 567,
+                "tasks": [],
+                "average_score": 0.6389872753509118
+            },
+            "contextual_formatted_text": {
+                "count": 63,
+                "num_samples": 972,
+                "tasks": [],
+                "average_score": 0.5354828682148746
+            },
+            "exact_text": {
+                "count": 57,
+                "num_samples": 876,
+                "tasks": [],
+                "average_score": 0.5603798147704114
+            },
+            "open_ended_output": {
+                "count": 51,
+                "num_samples": 986,
+                "tasks": [],
+                "average_score": 0.641307090345096
+            }
+        },
+        "input_num": {
+            "1-image": {
+                "count": 315,
+                "num_samples": 5215,
+                "tasks": [],
+                "average_score": 0.5772282223913566
+            }
+        },
+        "app": {
+            "Knowledge": {
+                "count": 77,
+                "num_samples": 1291,
+                "tasks": [],
+                "average_score": 0.6310985524386292
+            },
+            "Perception": {
+                "count": 82,
+                "num_samples": 1318,
+                "tasks": [],
+                "average_score": 0.6639417814110704
+            },
+            "Mathematics": {
+                "count": 30,
+                "num_samples": 497,
+                "tasks": [],
+                "average_score": 0.5009952551800727
+            },
+            "Information_Extraction": {
+                "count": 41,
+                "num_samples": 639,
+                "tasks": [],
+                "average_score": 0.6501282930308486
+            },
+            "Science": {
+                "count": 22,
+                "num_samples": 469,
+                "tasks": [],
+                "average_score": 0.5602180589214083
+            },
+            "Planning": {
+                "count": 44,
+                "num_samples": 712,
+                "tasks": [],
+                "average_score": 0.31397734418769446
+            },
+            "Coding": {
+                "count": 16,
+                "num_samples": 244,
+                "tasks": [],
+                "average_score": 0.5677003092903828
+            },
+            "Metrics": {
+                "count": 3,
+                "num_samples": 45,
+                "tasks": [],
+                "average_score": 0.626984126984127
+            }
+        }
+    }
+}

static/eval_results/SI/Gemini-Flash-2.0-exp/task_results.json

@@ -0,0 +1,2207 @@
+[
+    {
+        "name": "brand_logo_recognition_and_elaboration",
+        "score": 0.88,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 25
+    },
+    {
+        "name": "exchange_rate_estimate_plot",
+        "score": 0.9733571428571428,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "math_parity",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "traffic_future_prediction_from_line_plot",
+        "score": 0.6567894736842107,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "graph_chordless_cycle",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "youtube_video_info_parsing",
+        "score": 0.7976190476190476,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "super_clevr_scene_understanding",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "figureqa",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "face_keypoint_detection",
+        "score": 0.5787325110618333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "widerface_face_count_and_event_classification",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "average_humidity_estimate_plot",
+        "score": 0.8413333333333332,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "weather_info_parsing",
+        "score": 0.896825396825397,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "egocentric_analysis_single_image",
+        "score": 0.4444444444444444,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "waybill_number_sequence_extraction",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_maxflow",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "TV_show_info_parsing",
+        "score": 0.8015873015873015,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "insect_order_classification",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "electricity_plot_future_prediction",
+        "score": 0.8230157894736841,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "chemistry_exams_v",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "finance_table_understanding",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "funsd_document_qa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "vibe_eval_open",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "question_solution_solving",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_theory",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_analytic",
+        "score": 0.14285714285714285,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_length",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "algebra",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_puzzle_single_step",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "chess_winner_identification",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "physical_property_reasoning",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "humor_understand_caption_match",
+        "score": 0.7333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "coco_object_detection_by_query_property",
+        "score": 0.6698026360094582,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multilingual_game_info_parsing",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mnist_pattern",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "newspaper_page_parse_and_count",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "dvqa",
+        "score": 0.7894736842105263,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "science_basic_physics",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "electricity_future_prediction_from_table",
+        "score": 0.737017543859649,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "physics_exams_v",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "license_plate_recognition",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "snli_ve_visual_entailment",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "places365_scene_type_classification",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "3d_indoor_scene_text_bbox_selection",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_descriptive",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "top_rated_hotel_identification",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "science_molecule_chemistry",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "game_info_parsing",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "music_info_parsing",
+        "score": 0.6071428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "deciphering_oracle_bone",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multilingual_movie_info_parsing",
+        "score": 0.7346938775510203,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "iconqa_count_and_reasoning",
+        "score": 0.631578947368421,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "graph_connectivity",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "graph_shortest_path_planar",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "famous_building_recognition",
+        "score": 0.9375,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "signboard_identification",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_transformation",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "image_style_recognition",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "math_convexity_value_estimation",
+        "score": 0.5207276876255985,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "long_string_letter_recognition",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "3d_indoor_scene_text_bbox_prediction",
+        "score": 0.19380819004670155,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "movie_info_parsing",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "handwritten_math_expression_extraction",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_solid",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "animal_pose_estimation",
+        "score": 0.3668898610562918,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "single_person_pose_estimation",
+        "score": 0.298785583093911,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "human_relationship_reasoning",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_shortest_path_kamada_kawai",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_area",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "coco_person_detection",
+        "score": 0.6026048935518167,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chart_vqa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "hotel_booking_confirmation_parsing",
+        "score": 0.7071428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ili_ratio_future_prediction",
+        "score": 0.16085714285714284,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "nlvr2_two_image_compare_qa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "electricity_load_estimate_plot",
+        "score": 0.7926428571428572,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "tqa_textbook_qa",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "stock_info_parsing",
+        "score": 0.9663865546218489,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "math_exams_v",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "quizlet_question_solving",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "newspaper_ocr_in_query_box",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "mvsa_sentiment_classification",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "egocentric_spatial_reasoning",
+        "score": 0.5555555555555556,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "stock_price_future_prediction",
+        "score": 0.8637857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "Ad_count_detection",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "recover_masked_word_in_figure",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "polygon_interior_angles",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "web_action_grounding",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "latex_complex_formula_convertion",
+        "score": 0.29411764705882354,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "transit_map_intersection_points",
+        "score": 0.3898809523809524,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "arxiv_vqa",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_image_artifacts_indentification",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "song_title_identification_from_lyrics",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "actor_recognition_in_Movie",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bongard_problem",
+        "score": 0.42105263157894735,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "ascii_art_understanding",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "calendar_schedule_suggestion",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_reasoning_overlapped_circle",
+        "score": 0.75,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_barman",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "planning_screenshot_floortile",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "graph_isomorphism",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "code_programming_test_easy",
+        "score": 0.375,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 24
+    },
+    {
+        "name": "biology_exams_v",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "long_string_number_recognition",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "kvqa_knowledge_aware_qa",
+        "score": 0.42105263157894735,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "math_breakpoint",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "landmark_recognition_and_qa",
+        "score": 0.7111111111111111,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "code_execution",
+        "score": 0.8125,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 16
+    },
+    {
+        "name": "music_sheet_format_QA",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "annoying_word_search",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "interpret_force_perspective_illusion",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "healthcare_info_judgement",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_plot_position_relationship",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "map_diagram_qa",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "pmc_vqa_medical_image_qa",
+        "score": 0.5789473684210527,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "medical_blood_vessels_recognition",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "relative_depth_of_different_points",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "location_vqa",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "topological_sort",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mindmap_elements_parsing",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "scibench_fundamental_wo_solution",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 49
+    },
+    {
+        "name": "geometry_reasoning_nested_squares",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "font_recognition",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "mensa_iq_test",
+        "score": 0.4372549019607843,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "flowchart_code_generation",
+        "score": 0.5555555555555556,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "geometry_reasoning_count_line_intersections",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "stackoverflow_debug_QA",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "logical_reasoning_find_odd_one_out",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "circuit_diagram_understanding",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "web_action_prediction",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "signage_navigation",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "go_capture_stone",
+        "score": 0.13333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "webpage_code_understanding",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "monthly_weather_days_count",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_counting_lymphocytes",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "weather_map_climate_type_temperature_parsing",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "top_video_creator_identification",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "rebus",
+        "score": 0.43478260869565216,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 23
+    },
+    {
+        "name": "ishihara_test",
+        "score": 0.19285714285714287,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "paper_vqa",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "product_ocr_qa",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "geometry_reasoning_circled_letter",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Act_Web_Single",
+        "score": 0.026658939437844104,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "game_platform_support_identification",
+        "score": 0.9642857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Act_Mobile_swipe",
+        "score": 0.5319532782862981,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 13
+    },
+    {
+        "name": "mahjong",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "extract_webpage_headline",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_storage",
+        "score": 0.26666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "scibench_calculus_wo_solution",
+        "score": 0.4489795918367347,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 49
+    },
+    {
+        "name": "knowledge_graph_understanding",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "image_translation_en2cn",
+        "score": 0.4747283144053553,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "realworld_qa_en2cn",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "soccer_offside",
+        "score": 0.5555555555555556,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "planning_visual_storage",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "geometry_reasoning_grid",
+        "score": 0.75,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Act_Web_Multi",
+        "score": 0.23629435627595965,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chinese_idiom_recognition",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "relative_reflectance_of_different_regions",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "number_comparison",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "entertainment_web_game_style",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "orchestra_score_recognition",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_blocksworld",
+        "score": 0.3333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "icon_arithmetic_puzzle",
+        "score": 0.6785714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_grippers",
+        "score": 0.4,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "character_recognition_in_TV_shows",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "highest_discount_game_price_identification",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "remaining_playback_time_calculation",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_cell_recognition",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_find_legal_moves",
+        "score": 0.20779353910574844,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "distinguish_ai_generated_image",
+        "score": 0.8421052631578947,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "autonomous_driving_scene_analysis",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "counting_single_image",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMMU_pro_exam_screenshot",
+        "score": 0.40404040404040403,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 99
+    },
+    {
+        "name": "GUI_Act_Mobile_tap",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "road_map_find_highway_between_two_place",
+        "score": 0.8235294117647058,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "clevrer_physics",
+        "score": 0.45,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 20
+    },
+    {
+        "name": "chess_sygyzy_endgames",
+        "score": 0.15739022881880024,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "llavaguard",
+        "score": 0.5357142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMMU_physics_chemistry_selected",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "medical_multi_organ_segmentation_rater",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "cultural_vqa",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "logical_reasoning_fit_pattern",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "planning_screenshot_tyreworld",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "music_sheet_note_count",
+        "score": 0.058823529411764705,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "planning_screenshot_termes",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "hashtag_recommendation",
+        "score": 0.9166666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_europe",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_americas",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_distance",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_count",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_depth",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "adapted_cvbench_relation",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "symbolic_graphics_programs_computer_aided_design",
+        "score": 0.2857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "symbolic_graphics_programs_scalable_vector_graphics",
+        "score": 0.1111111111111111,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 18
+    },
+    {
+        "name": "table_understanding_complex_question_answering",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "table_understanding_fact_verification",
+        "score": 0.9047619047619048,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "panel_images_multi_question",
+        "score": 0.8095238095238095,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_asia",
+        "score": 0.9000000000000001,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "panel_images_single_question",
+        "score": 1.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multiple_states_identify_africa",
+        "score": 0.8142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMSoc_Misinformation_GossipCop",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "MMSoc_HatefulMemes",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "poetry_petrarchian_sonnet_optional_meter",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "MMSoc_Memotion",
+        "score": 0.6117647058823531,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "poetry_haiku",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "MMSoc_Misinformation_PolitiFact",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "poetry_shakespearean_sonnet",
+        "score": 0.06666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "poetry_acrostic_alliteration",
+        "score": 0.7333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "screenshot_lighteval_math",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "poetry_acrostic",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "poetry_limerick",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "screenshot_theoremqa",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "poetry_custom_rhyming_scheme",
+        "score": 0.2,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "text_entity_replace",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "background_change",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "face_attribute_edit",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "face_swap",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "text_style",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "number_puzzle_sudoku",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "out_of_context",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "clip_stable_diffusion_generate",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "veracity",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "counterfactual_arithmetic",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "maze_2d_8x8",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "shape_composition_shapes",
+        "score": 0.47176870748299315,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "shape_composition_colours",
+        "score": 0.4498299319727891,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "number_puzzle_kakuro_5x5",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "autorater_artifact",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "autorater_artifact_reason",
+        "score": 0.6666666666666666,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "chess_puzzles_crushing",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_amazon",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_puzzles_checkmate",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_instagram",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "chess_puzzles_equality",
+        "score": 0.0,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "app_layout_understanding_zoom",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "app_layout_understanding_notes",
+        "score": 0.5,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_word",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_twitter",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_iphone_settings",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_youtube",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_leetcode",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_ppt",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_tiktok",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "app_layout_understanding_alipay",
+        "score": 0.8235294117647058,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 17
+    },
+    {
+        "name": "app_layout_understanding_excel",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_employer_plain",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_article_journal",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_experience_plain",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_markdown",
+        "score": 0.9285714285714286,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_math_text_latex",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_latex",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_school_plain",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_article_authors",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_html",
+        "score": 0.7142857142857143,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_resume_skill_plain",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_table_to_csv",
+        "score": 0.6428571428571429,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "crossword_mini_5x5",
+        "score": 0.6714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ocr_math_equation",
+        "score": 0.42857142857142855,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "contain_repeat_length",
+        "score": 0.4666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "contain_position_length",
+        "score": 0.8666666666666667,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "pictionary_skribbl_io",
+        "score": 0.3,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 20
+    },
+    {
+        "name": "pictionary_doodle_guess",
+        "score": 0.8,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "pictionary_genai_output_chinese",
+        "score": 0.35714285714285715,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "pictionary_cartoon_drawing_guess",
+        "score": 0.8571428571428571,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "contain_length",
+        "score": 0.5333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "pictionary_chinese_food_img2en",
+        "score": 0.7857142857142857,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "contain_contain_length",
+        "score": 0.9333333333333333,
+        "eval_type": "rule",
+        "num_demo": 0,
+        "num_query": 15
+    },
+    {
+        "name": "reward_models_i2t_reward",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "memorization_chinese_celebrity",
+        "score": 0.5714285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "memorization_papers",
+        "score": 0.6,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "memorization_famous_treaty",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "memorization_indian_celebrity",
+        "score": 0.8214285714285714,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "research_website_parsing_blogpost",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "research_website_parsing_publication",
+        "score": 0.07142857142857142,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "research_website_parsing_homepage",
+        "score": 0.21428571428571427,
+        "eval_type": "rule",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "ascii_art_30",
+        "score": 0.21428571428571427,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "humor_explanation",
+        "score": 0.8533333333333335,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 15
+    },
+    {
+        "name": "science_figure_explanation",
+        "score": 0.872413793103448,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "vibe_eval_phrase",
+        "score": 0.7571428571428573,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "traffic_accident_analysis",
+        "score": 0.6714285714285714,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "figurative_speech_explanation",
+        "score": 0.8344827586206895,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "table2latex_complex",
+        "score": 0.8111111111111111,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 9
+    },
+    {
+        "name": "unusual_images",
+        "score": 0.889655172413793,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "art_explanation",
+        "score": 0.4655172413793103,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "ocr_open_ended_qa",
+        "score": 0.7965517241379308,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "bar_chart_interpretation",
+        "score": 0.6655172413793102,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "scibench_w_solution_open_ended",
+        "score": 0.618,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 25
+    },
+    {
+        "name": "GUI_Chat_Hard",
+        "score": 0.4115384615384616,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 26
+    },
+    {
+        "name": "image_humor_understanding",
+        "score": 0.882758620689655,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "defeasible_reasoning",
+        "score": 0.8620689655172413,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "funny_image_title",
+        "score": 0.6499999999999998,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "tweets_captioning",
+        "score": 0.5499999999999999,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "graph_interpretation",
+        "score": 0.8413793103448274,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "meme_explain",
+        "score": 0.8857142857142858,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "guess_image_generation_prompt",
+        "score": 0.8263157894736844,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "visualization_with_code",
+        "score": 0.6714285714285715,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "iq_test_open_ended",
+        "score": 0.6689655172413791,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 29
+    },
+    {
+        "name": "electrocardiogram",
+        "score": 0.3857142857142857,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "image_captioning_with_additional_requirements",
+        "score": 0.9285714285714287,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "docci_image_description_long",
+        "score": 0.7928571428571428,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "GUI_Chat_Easy",
+        "score": 0.5769230769230769,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 26
+    },
+    {
+        "name": "bridge_strategies_advanced",
+        "score": 0.15714285714285717,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bridge_strategies_worldclass",
+        "score": 0.08571428571428572,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "bridge_strategies_expert",
+        "score": 0.19999999999999998,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Spanish",
+        "score": 0.7428571428571429,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Japanese",
+        "score": 0.6857142857142857,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_French",
+        "score": 0.7357142857142858,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Arabic",
+        "score": 0.7000000000000001,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_Russian",
+        "score": 0.6857142857142857,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "multi_lingual_Ruozhiba_expalnation_English",
+        "score": 0.7285714285714285,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "table_understanding_fetaqa",
+        "score": 0.5928571428571429,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 14
+    },
+    {
+        "name": "red_teaming_celebrity",
+        "score": 0.8550000000000001,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "red_teaming_captcha",
+        "score": 0.13157894736842107,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "red_teaming_jailbreak",
+        "score": 0.5850000000000001,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "red_teaming_visualmisleading",
+        "score": 0.8789473684210528,
+        "eval_type": "llm",
+        "num_demo": 1,
+        "num_query": 19
+    },
+    {
+        "name": "red_teaming_racial",
+        "score": 0.7600000000000001,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    },
+    {
+        "name": "red_teaming_politics",
+        "score": 0.695,
+        "eval_type": "llm",
+        "num_demo": 0,
+        "num_query": 20
+    }
+]