Add new benchmarks; Several improvements

app.py

@@ -1,8 +1,10 @@
 import json
-from datetime import datetime, date
+from datetime import datetime, date, timedelta
 
 import gradio as gr
 import plotly.graph_objects as go
+from scipy.optimize import curve_fit
+import numpy as np
 
 
 def create_big_five_capex_plot() -> go.Figure:
@@ -11,8 +13,8 @@ def create_big_five_capex_plot() -> go.Figure:
         data = [json.loads(line) for line in file if line.strip()]
 
     quarters: list[str] = [entry["Quarter"] for entry in data]
-    companies = ['Microsoft', 'Google', 'Meta', 'Apple', 'Amazon']
-    colors = ['#80bb00', '#ee161f', '#0065e3', '#000000', '#ff6200']
+    companies = ['Microsoft', 'Google', 'Meta', 'Amazon']
+    colors = ['#80bb00', '#ee161f', '#0065e3', '#ff6200']
 
     x_positions = list(range(len(quarters)))
 
@@ -29,7 +31,7 @@ def create_big_five_capex_plot() -> go.Figure:
     fig = go.Figure(data=traces)
     fig.update_layout(
         barmode="stack",
-        title="Capital Expenditures of the Big Five Tech Companies in Millions of USD per Quarter",
+        title="Capital Expenditures of Amazon, Meta, Google and Microsoft in Millions of USD per Quarter",
         xaxis_title="Quarter",
         yaxis_title="Capital Expenditures (Millions USD)",
         xaxis=dict(
@@ -37,7 +39,7 @@ def create_big_five_capex_plot() -> go.Figure:
             tickvals=x_positions,
             ticktext=quarters
         ),
-        height=600
+        height=800
     )
 
     # Calculate the x position for the vertical dotted line.
@@ -86,12 +88,14 @@ def create_big_five_capex_plot() -> go.Figure:
 
 def create_simple_plot(data_path: str,
                        name: str,
+                       subtitle: str,
                        start_date: datetime, end_date: datetime,
-                       min_value: int = 0, max_value: int = 100) -> go.Figure:
-    simple_bench_leaderboard = []
+                       min_value: int = 0, max_value: int = 100,
+                       labeled_horizontal_lines: dict[str, float] = None) -> go.Figure:
+    leaderboard = []
     with open(data_path, 'r') as file:
         for line in file:
-            simple_bench_leaderboard.append(json.loads(line))
+            leaderboard.append(json.loads(line))
 
     models = []
     with open("models.jsonl", 'r') as file:
@@ -99,7 +103,7 @@ def create_simple_plot(data_path: str,
             models.append(json.loads(line))
 
     data = []
-    for entry in simple_bench_leaderboard:
+    for entry in leaderboard:
         model_name = entry['model']
         score = entry['score']
         model_info = next((m for m in models if m['Name'] == model_name), None)
@@ -142,8 +146,8 @@ def create_simple_plot(data_path: str,
             ))
 
     fig.update_layout(
-        title=f'{name} Over Time',
-        xaxis_title='Release Date',
+        title=f'{name} Over Time<br><sup>{subtitle}</sup>',
+        xaxis_title='Publication or Release Date',
         yaxis_title=name,
         hovermode='x unified',
         xaxis=dict(
@@ -156,67 +160,237 @@ def create_simple_plot(data_path: str,
         height=800
     )
 
+    if labeled_horizontal_lines:
+        for label, y_value in labeled_horizontal_lines.items():
+            fig.add_hline(
+                y=y_value,
+                line_dash="dot",
+                line_color="black",
+                annotation_text=label,
+                annotation_position="right",
+                annotation=dict(
+                    font_size=12,
+                    font_color="black",
+                    xanchor="left",
+                    yanchor="middle",
+                    xshift=10
+                )
+            )
+
     return fig
 
 
 with gr.Blocks() as demo:
     with gr.Tab("System Performance Over Time"):
-        with gr.Tab("ARC-AGI-Pub") as arc_agi_tab:
-            arc_agi_plot: gr.Plot = gr.Plot()
-        with gr.Tab("Simple Bench") as simple_bench_tab:
+        with gr.Tab("Legend"):
+            legend_markdown: gr.Markdown = gr.Markdown(
+                value="""
+## Benchmarks and Top Scores
+
+| Benchmark | Top Score |
+|-----------|-----------|
+| BigCodeBench | 🟠 36% |
+| Simple Bench | 🟠 42% |
+| PlanBench | 🟠 53% |
+| GAIA | 🟡 65% |
+| ARC-AGI-Pub (Semi-Private Eval) | 🟡 76% |
+| GPQA | 🟡 76% |
+| ZebraLogic | 🟡 81% |
+| ARC-AGI-Pub (Public Eval) | 🟡 83% |
+| ZeroEval | 🟡 86% |
+| MATH-L5 | 🟡 89% |
+| MMLU-Redux | 🟢 93% |
+| CRUX | 🟢 96% |
+
+## Colors
+
+| Color | Score Range |
+|-------|------------|
+| 🔴 Red | Below 30% |
+| 🟠 Orange | 30% to 60% |
+| 🟡 Yellow | 60% to 90% |
+| 🟢 Green | Above 90% |"""
+            )
+        with gr.Tab("🟠 BigCodeBench") as bigcodebench_tab:
+            bigcodebench_plot: gr.Plot = gr.Plot()
+            bigcodebench_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [BigCodeBench Leaderboard](https://bigcode-bench.github.io/)"""
+            )
+        with gr.Tab("🟠 Simple Bench") as simple_bench_tab:
             simple_bench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("PlanBench") as planbench_tab:
+            simple_bench_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [SimpleBench Leaderboard](https://simple-bench.com/)"""
+            )
+        with gr.Tab("🟠 PlanBench") as planbench_tab:
             planbench_plot: gr.Plot = gr.Plot()
             planbench_markdown: gr.Markdown = gr.Markdown(
                 value="""Source: [Valmeekam et al. 2024](https://arxiv.org/abs/2409.13373)"""
             )
-        with gr.Tab("Codeforces") as codeforces_tab:
-            with gr.Tab("General-Purpose Systems"):
-                codeforces_plot: gr.Plot = gr.Plot()
-        with gr.Tab("BigCodeBench", interactive=False):
-            bigcodebench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("GAIA", interactive=False):
+        with gr.Tab("🟡 GAIA") as gaia_tab:
             gaia_plot: gr.Plot = gr.Plot()
-        with gr.Tab("GPQA", interactive=False):
+            gaia_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [GAIA Leaderboard](https://huggingface.co/spaces/gaia-benchmark/leaderboard)"""
+            )
+        with gr.Tab("🟡 ARC-AGI-Pub") as arc_agi_tab:
+            with gr.Tab("🟡 Semi-Private Eval") as arc_agi_semi_private_eval_tab:
+                arc_agi_semi_private_eval_plot: gr.Plot = gr.Plot()
+            with gr.Tab("🟡 Public Eval") as arc_agi_public_eval_tab:
+                arc_agi_public_eval_plot: gr.Plot = gr.Plot()
+            arc_agi_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [ARC Prize 2024](https://arcprize.org/2024-results)"""
+            )
+        with gr.Tab("🟡 GPQA") as gpqa_tab:
             gpqa_plot: gr.Plot = gr.Plot()
-        with gr.Tab("HumanEval", interactive=False):
-            humaneval_plot: gr.Plot = gr.Plot()
-        with gr.Tab("Chatbot Arena", interactive=False):
-            chatbot_arena_plot: gr.Plot = gr.Plot()
-        with gr.Tab("MATH", interactive=False):
-            math_plot: gr.Plot = gr.Plot()
-        with gr.Tab("OpenCompass", interactive=False):
+            gpqa_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [Epoch AI Benchmarking Dashboard](https://epoch.ai/data/ai-benchmarking-dashboard)"""
+            )
+        with gr.Tab("🟡 ZebraLogic") as zeroeval_zebralogic_tab:
+            zeroeval_zebralogic_plot: gr.Plot = gr.Plot()
+            zeroeval_zebralogic_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)"""
+            )
+        with gr.Tab("🟡 ZeroEval") as zeroeval_average_tab:
+            zeroeval_average_plot: gr.Plot = gr.Plot()
+            zeroeval_average_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)"""
+            )
+        with gr.Tab("🟡 MATH-L5") as zeroeval_math_l5_tab:
+            zeroeval_math_l5_plot: gr.Plot = gr.Plot()
+            zeroeval_math_l5_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)"""
+            )
+        with gr.Tab("🟢 MMLU-Redux") as zeroeval_mmlu_redux_tab:
+            zeroeval_mmlu_redux_plot: gr.Plot = gr.Plot()
+            zeroeval_mmlu_redux_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)"""
+            )
+        with gr.Tab("🟢 CRUX") as zeroeval_crux_tab:
+            zeroeval_crux_plot: gr.Plot = gr.Plot()
+            zeroeval_crux_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)"""
+            )
+        with gr.Tab("Codeforces") as codeforces_tab:
+            codeforces_plot: gr.Plot = gr.Plot()
+        with gr.Tab("OpenCompass", visible=False):
             opencompass_plot: gr.Plot = gr.Plot()
-        with gr.Tab("SWE-bench", interactive=False):
+            opencompass_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [OpenCompass LLM Leaderboard](https://huggingface.co/spaces/opencompass/opencompass-llm-leaderboard)"""
+            )
+        with gr.Tab("SWE-bench", visible=False):
             swe_bench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("WebArena", interactive=False):
+            swe_bench_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [SWE-bench Leaderboard](https://www.swebench.com/)"""
+            )
+        with gr.Tab("WebArena", visible=False):
             webarena_plot: gr.Plot = gr.Plot()
-        with gr.Tab("ZeroEval", interactive=False):
-            zeroeval_plot: gr.Plot = gr.Plot()
+            webarena_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [X-WebArena-Leaderboard](https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ)"""
+            )
     with gr.Tab("Finance") as finance_tab:
-        with gr.Tab("Big Five Capex") as big_five_capex_tab:
+        with gr.Tab("Big Tech Capex") as big_five_capex_tab:
             big_five_capex_plot: gr.Plot = gr.Plot()
-        with gr.Tab("NVIDIA Revenue", interactive=False) as nvidia_revenue:
+        with gr.Tab("NVIDIA Revenue", visible=False) as nvidia_revenue:
             nvidia_revenue_plot: gr.Plot = gr.Plot()
     big_five_capex_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot)
-    finance_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot)
+    arc_agi_public_eval_tab.select(fn=create_simple_plot,
+                                   inputs=[gr.State("arc_agi_leaderboard.jsonl"),
+                                           gr.State("ARC-AGI-Pub Score (Public Eval, $20 Compute Budget per Task, General-Purpose Systems)"),
+                                           gr.State("\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"),
+                                           gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)),
+                                           gr.State(0), gr.State(100),
+                                           gr.State({"Humans\n(LeGris et al. 2024)": 64.2})],
+                                   outputs=arc_agi_public_eval_plot)
     arc_agi_tab.select(fn=create_simple_plot,
-                       inputs=[gr.State("arc_agi_leaderboard.jsonl"), gr.State("ARC-AGI-Pub (Public Eval) Score"),
-                               gr.State(date(2024, 5, 13)), gr.State(date(2024, 12, 20))],
-                       outputs=arc_agi_plot)
+                       inputs=[gr.State("arc_agi_semi_private_eval_leaderboard.jsonl"),
+                               gr.State("ARC-AGI-Pub Score (Semi-Private Eval, $20 Compute Budget per Task, General-Purpose Systems)"),
+                               gr.State("\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"),
+                               gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)),
+                               gr.State(0), gr.State(100),
+                               gr.State({"MTurkers": 77})],
+                       outputs=arc_agi_semi_private_eval_plot)
+    arc_agi_semi_private_eval_tab.select(fn=create_simple_plot,
+                                         inputs=[gr.State("arc_agi_semi_private_eval_leaderboard.jsonl"),
+                                                 gr.State("ARC-AGI-Pub Score (Semi-Private Eval, $20 Compute Budget per Task, General-Purpose Systems)"),
+                                                 gr.State("\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"),
+                                                 gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)),
+                                                 gr.State(0), gr.State(100),
+                                                 gr.State({"MTurkers": 77})],
+                                         outputs=arc_agi_semi_private_eval_plot)
+    finance_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot)
     simple_bench_tab.select(fn=create_simple_plot,
-                            inputs=[gr.State("simple_bench_leaderboard.jsonl"), gr.State("Simple Bench Score"),
-                                    gr.State(date(2023, 6, 13)), gr.State(date(2024, 8, 14))],
+                            inputs=[gr.State("simple_bench_leaderboard.jsonl"),
+                                    gr.State("Simple Bench Score"),
+                                    gr.State("\"multiple-choice text benchmark [...] [including] over 200 questions covering spatio-temporal reasoning, social intelligence, and what we call linguistic adversarial robustness\" (Philip & Hemang, 2024)"),
+                                    gr.State(date(2024, 4, 1)), gr.State(date(2025, 1, 1)),
+                                    gr.State(0), gr.State(100),
+                                    gr.State({"Humans": 83.7})],
                             outputs=simple_bench_plot)
     codeforces_tab.select(fn=create_simple_plot,
-                          inputs=[gr.State("codeforces_leaderboard.jsonl"), gr.State("Codeforces (Elo Rating)"),
-                                  gr.State(date(2024, 5, 13)), gr.State(date(2024, 12, 20)),
-                                  gr.State(800), gr.State(3000)],
+                          inputs=[gr.State("codeforces_leaderboard.jsonl"),
+                                  gr.State("Codeforces Rating"),
+                                  gr.State("\"[Codeforces] is a platform where [programming] contests are held regularly, the participant's skills are reflected by their rating [...] The rating is a modification of Elo rating\" (Mirzayanov, 2011)"),
+                                  gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)),
+                                  gr.State(0), gr.State(4000),
+                                  gr.State({"Pupil": 1200, "Specialist": 1400, "Expert": 1600, "Candidate Master": 1900, "Master": 2100, "International Master": 2300, "Grandmaster": 2400, "International Grandmaster": 2600, "Legendary Grandmaster": 3000})],
                           outputs=codeforces_plot)
     planbench_tab.select(fn=create_simple_plot,
-                         inputs=[gr.State("planbench_leaderboard.jsonl"), gr.State("PlanBench (Mystery Blocksworld, 0-shot) Score"),
-                                 gr.State(date(2023, 3, 14)), gr.State(date(2024, 9, 23))],
+                         inputs=[gr.State("planbench_leaderboard.jsonl"),
+                                 gr.State("PlanBench Score (Mystery Blocksworld, 0-shot)"),
+                                 gr.State("\"benchmark suite based on the kinds of domains used in the automated planning community [...] to test the capabilities of LLMs in planning or reasoning about actions and change.\" (Valmeekam et al. 2022)"),
+                                 gr.State(date(2023, 3, 1)), gr.State(date(2024, 9, 20))],
                          outputs=planbench_plot)
+    bigcodebench_tab.select(fn=create_simple_plot,
+                            inputs=[gr.State("bigcodebench_hard_average_leaderboard.jsonl"),
+                                    gr.State("BigCodeBench Score (Hard, Average of Complete and Instruct)"),
+                                    gr.State("\"benchmark that challenges LLMs to invoke multiple function calls as tools from 139 libraries and 7 domains for 1,140 fine-grained tasks\" (Zhuo et al. 2024)"),
+                                    gr.State(date(2023, 6, 1)), gr.State(date(2025, 1, 1))],
+                            outputs=bigcodebench_plot)
+    gaia_tab.select(fn=create_simple_plot,
+                    inputs=[gr.State("gaia_leaderboard.jsonl"),
+                            gr.State("General AI Assistants (GAIA) Benchmark Score (Test Set, Average)"),
+                            gr.State("\"real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency\" (Mialon et al. 2023)"),
+                            gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1)),
+                            gr.State(0), gr.State(100),
+                            gr.State({"Humans": 92})],
+                    outputs=gaia_plot)
+    gpqa_tab.select(fn=create_simple_plot,
+                    inputs=[gr.State("gpqa_leaderboard.jsonl"),
+                            gr.State("Graduate-Level Google-Proof Q&A (GPQA) Benchmark Score"),
+                            gr.State("\"challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry [that] are high-quality and extremely difficult\" (Rein et al. 2023)"),
+                            gr.State(date(2023, 6, 1)), gr.State(date(2025, 1, 1)),
+                            gr.State(25), gr.State(100),
+                            gr.State({"Highly skilled non-expert validators": 34, "PhD-level domain experts": 65})],
+                    outputs=gpqa_plot)
+    zeroeval_average_tab.select(fn=create_simple_plot,
+                                inputs=[gr.State("zeroeval_average_leaderboard.jsonl"),
+                                        gr.State("ZeroEval Average (MMLU-Redux, ZebraLogic, CRUX and MATH-5) Score"),
+                                        gr.State("\"a simple unified framework for evaluating language models on various tasks\" (Ai2, 2024)"),
+                                        gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))],
+                                outputs=zeroeval_average_plot)
+    zeroeval_mmlu_redux_tab.select(fn=create_simple_plot,
+                                   inputs=[gr.State("zeroeval_mmlu_redux_leaderboard.jsonl"),
+                                           gr.State("ZeroEval MMLU-Redux (Massive Multitask Language Understanding) Score"),
+                                           gr.State("\"knowledge reasoning\" (Ai2, 2024); \"subset of 3,000 manually re-annotated questions across 30 MMLU subjects\" (Gema et al. 2024)"),
+                                           gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))],
+                                   outputs=zeroeval_mmlu_redux_plot)
+    zeroeval_zebralogic_tab.select(fn=create_simple_plot,
+                                   inputs=[gr.State("zeroeval_zebralogic_leaderboard.jsonl"),
+                                           gr.State("ZeroEval ZebraLogic Score"),
+                                           gr.State("\"logical reasoning\" (Ai2, 2024); \"Each example is a Logic Grid Puzzle [...] often used to test humans' logical reasoning abilities\" (Lin, 2024)"),
+                                           gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))],
+                                   outputs=zeroeval_zebralogic_plot)
+    zeroeval_crux_tab.select(fn=create_simple_plot,
+                             inputs=[gr.State("zeroeval_crux_leaderboard.jsonl"),
+                                     gr.State("ZeroEval CRUX (Code Reasoning, Understanding, and eXecution Evaluation) Score"),
+                                     gr.State("\"code reasoning\" (Ai2, 2024); \"benchmark consisting of 800 Python functions (3-13 lines). Each function comes with [...] two natural tasks: input prediction and output prediction.\" (Gu et al. 2024)"),
+                                     gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))],
+                             outputs=zeroeval_crux_plot)
+    zeroeval_math_l5_tab.select(fn=create_simple_plot,
+                                inputs=[gr.State("zeroeval_math_l5_leaderboard.jsonl"),
+                                        gr.State("ZeroEval MATH-L5 (Difficulty Level 5 of MATH) Score"),
+                                        gr.State("\"math reasoning\" (Ai2, 2024); \"dataset of 12,500 challenging competition mathematics problems. [...] a subject’s hardest problems are assigned a difficulty level of ‘5.’\" (Hendrycks et al. 2021)"),
+                                        gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))],
+                                outputs=zeroeval_math_l5_plot)
 
 
 if __name__ == "__main__":

arc_agi_semi_private_eval_leaderboard.jsonl

@@ -0,0 +1,6 @@
+{"model": "o3", "score": 75.7}
+{"model": "o1-2024-12-17", "score": 32}
+{"model": "o1-preview-2024-09-12", "score": 18}
+{"model": "claude-3-5-sonnet-20240620", "score": 14}
+{"model": "gpt-4o-2024-05-13", "score": 5}
+{"model": "gemini-1.5-pro-001", "score": 4.5}

big_five_capex.jsonl

@@ -1,40 +1,40 @@
-{"Quarter": "2015 Q1", "Microsoft": 1391, "Google": 2927, "Meta": 502, "Apple": 2369, "Amazon": 871}
-{"Quarter": "2015 Q2", "Microsoft": 1781, "Google": 2515, "Meta": 549, "Apple": 2043, "Amazon": 1213}
-{"Quarter": "2015 Q3", "Microsoft": 1356, "Google": 2406, "Meta": 780, "Apple": 3618, "Amazon": 1195}
-{"Quarter": "2015 Q4", "Microsoft": 2024, "Google": 2102, "Meta": 692, "Apple": 3612, "Amazon": 1309}
-{"Quarter": "2016 Q1", "Microsoft": 2308, "Google": 2444, "Meta": 1132, "Apple": 2336, "Amazon": 1179}
-{"Quarter": "2016 Q2", "Microsoft": 2655, "Google": 2136, "Meta": 995, "Apple": 2809, "Amazon": 1711}
-{"Quarter": "2016 Q3", "Microsoft": 2163, "Google": 2554, "Meta": 1095, "Apple": 3977, "Amazon": 1841}
-{"Quarter": "2016 Q4", "Microsoft": 1988, "Google": 3078, "Meta": 1269, "Apple": 3334, "Amazon": 3073}
-{"Quarter": "2017 Q1", "Microsoft": 1695, "Google": 2508, "Meta": 1271, "Apple": 2975, "Amazon": 2148}
-{"Quarter": "2017 Q2", "Microsoft": 2283, "Google": 2831, "Meta": 1444, "Apple": 2277, "Amazon": 3113}
-{"Quarter": "2017 Q3", "Microsoft": 2132, "Google": 3538, "Meta": 1755, "Apple": 3865, "Amazon": 3074}
-{"Quarter": "2017 Q4", "Microsoft": 2586, "Google": 4307, "Meta": 2263, "Apple": 2810, "Amazon": 3619}
-{"Quarter": "2018 Q1", "Microsoft": 2934, "Google": 7299, "Meta": 2812, "Apple": 4195, "Amazon": 3098}
-{"Quarter": "2018 Q2", "Microsoft": 3980, "Google": 5477, "Meta": 3460, "Apple": 3267, "Amazon": 3243}
-{"Quarter": "2018 Q3", "Microsoft": 3602, "Google": 5282, "Meta": 3342, "Apple": 3041, "Amazon": 3352}
-{"Quarter": "2018 Q4", "Microsoft": 3707, "Google": 7081, "Meta": 4301, "Apple": 3355, "Amazon": 3734}
-{"Quarter": "2019 Q1", "Microsoft": 2565, "Google": 4638, "Meta": 3837, "Apple": 2363, "Amazon": 3290}
-{"Quarter": "2019 Q2", "Microsoft": 4051, "Google": 6126, "Meta": 3633, "Apple": 2000, "Amazon": 3562}
-{"Quarter": "2019 Q3", "Microsoft": 3385, "Google": 6732, "Meta": 3532, "Apple": 2777, "Amazon": 4697}
-{"Quarter": "2019 Q4", "Microsoft": 3545, "Google": 6052, "Meta": 4100, "Apple": 2107, "Amazon": 5312}
-{"Quarter": "2020 Q1", "Microsoft": 3767, "Google": 6005, "Meta": 3558, "Apple": 1853, "Amazon": 6795}
-{"Quarter": "2020 Q2", "Microsoft": 4744, "Google": 5391, "Meta": 3255, "Apple": 1565, "Amazon": 7459}
-{"Quarter": "2020 Q3", "Microsoft": 4907, "Google": 5406, "Meta": 3689, "Apple": 1784, "Amazon": 11063}
-{"Quarter": "2020 Q4", "Microsoft": 4174, "Google": 5479, "Meta": 4613, "Apple": 3500, "Amazon": 14823}
-{"Quarter": "2021 Q1", "Microsoft": 5089, "Google": 5942, "Meta": 4303, "Apple": 2269, "Amazon": 12082}
-{"Quarter": "2021 Q2", "Microsoft": 6452, "Google": 5496, "Meta": 4641, "Apple": 2093, "Amazon": 14288}
-{"Quarter": "2021 Q3", "Microsoft": 5810, "Google": 6819, "Meta": 4346, "Apple": 3223, "Amazon": 15748}
-{"Quarter": "2021 Q4", "Microsoft": 5865, "Google": 6383, "Meta": 5400, "Apple": 2803, "Amazon": 18935}
-{"Quarter": "2022 Q1", "Microsoft": 5340, "Google": 9786, "Meta": 5441, "Apple": 2514, "Amazon": 14951}
-{"Quarter": "2022 Q2", "Microsoft": 6871, "Google": 6828, "Meta": 7572, "Apple": 2102, "Amazon": 15724}
-{"Quarter": "2022 Q3", "Microsoft": 6283, "Google": 7276, "Meta": 9375, "Apple": 3289, "Amazon": 16378}
-{"Quarter": "2022 Q4", "Microsoft": 6274, "Google": 7595, "Meta": 9043, "Apple": 3787, "Amazon": 16592}
-{"Quarter": "2023 Q1", "Microsoft": 6607, "Google": 6289, "Meta": 6823, "Apple": 2916, "Amazon": 14207}
-{"Quarter": "2023 Q2", "Microsoft": 8943, "Google": 6888, "Meta": 6134, "Apple": 2093, "Amazon": 11455}
-{"Quarter": "2023 Q3", "Microsoft": 9917, "Google": 8055, "Meta": 6543, "Apple": 2163, "Amazon": 12479}
-{"Quarter": "2023 Q4", "Microsoft": 9735, "Google": 11019, "Meta": 7665, "Apple": 2392, "Amazon": 14588}
-{"Quarter": "2024 Q1", "Microsoft": 10952, "Google": 12012, "Meta": 6400, "Apple": 1996, "Amazon": 14925}
-{"Quarter": "2024 Q2", "Microsoft": 13873, "Google": 13186, "Meta": 8173, "Apple": 2151, "Amazon": 17620}
-{"Quarter": "2024 Q3", "Microsoft": 14923, "Google": 13016, "Meta": 8258, "Apple": 0, "Amazon": 22620}
-{"Quarter": "2024 Q4", "Microsoft": 15804, "Google": 14276, "Meta": 14425, "Apple": 0, "Amazon": 27834}
+{"Quarter": "2015 Q1", "Microsoft": 1391, "Google": 2927, "Meta": 502, "Amazon": 871}
+{"Quarter": "2015 Q2", "Microsoft": 1781, "Google": 2515, "Meta": 549, "Amazon": 1213}
+{"Quarter": "2015 Q3", "Microsoft": 1356, "Google": 2406, "Meta": 780, "Amazon": 1195}
+{"Quarter": "2015 Q4", "Microsoft": 2024, "Google": 2102, "Meta": 692, "Amazon": 1309}
+{"Quarter": "2016 Q1", "Microsoft": 2308, "Google": 2444, "Meta": 1132, "Amazon": 1179}
+{"Quarter": "2016 Q2", "Microsoft": 2655, "Google": 2136, "Meta": 995, "Amazon": 1711}
+{"Quarter": "2016 Q3", "Microsoft": 2163, "Google": 2554, "Meta": 1095, "Amazon": 1841}
+{"Quarter": "2016 Q4", "Microsoft": 1988, "Google": 3078, "Meta": 1269, "Amazon": 3073}
+{"Quarter": "2017 Q1", "Microsoft": 1695, "Google": 2508, "Meta": 1271, "Amazon": 2148}
+{"Quarter": "2017 Q2", "Microsoft": 2283, "Google": 2831, "Meta": 1444, "Amazon": 3113}
+{"Quarter": "2017 Q3", "Microsoft": 2132, "Google": 3538, "Meta": 1755, "Amazon": 3074}
+{"Quarter": "2017 Q4", "Microsoft": 2586, "Google": 4307, "Meta": 2263, "Amazon": 3619}
+{"Quarter": "2018 Q1", "Microsoft": 2934, "Google": 7299, "Meta": 2812, "Amazon": 3098}
+{"Quarter": "2018 Q2", "Microsoft": 3980, "Google": 5477, "Meta": 3460, "Amazon": 3243}
+{"Quarter": "2018 Q3", "Microsoft": 3602, "Google": 5282, "Meta": 3342, "Amazon": 3352}
+{"Quarter": "2018 Q4", "Microsoft": 3707, "Google": 7081, "Meta": 4301, "Amazon": 3734}
+{"Quarter": "2019 Q1", "Microsoft": 2565, "Google": 4638, "Meta": 3837, "Amazon": 3290}
+{"Quarter": "2019 Q2", "Microsoft": 4051, "Google": 6126, "Meta": 3633, "Amazon": 3562}
+{"Quarter": "2019 Q3", "Microsoft": 3385, "Google": 6732, "Meta": 3532, "Amazon": 4697}
+{"Quarter": "2019 Q4", "Microsoft": 3545, "Google": 6052, "Meta": 4100, "Amazon": 5312}
+{"Quarter": "2020 Q1", "Microsoft": 3767, "Google": 6005, "Meta": 3558, "Amazon": 6795}
+{"Quarter": "2020 Q2", "Microsoft": 4744, "Google": 5391, "Meta": 3255, "Amazon": 7459}
+{"Quarter": "2020 Q3", "Microsoft": 4907, "Google": 5406, "Meta": 3689, "Amazon": 11063}
+{"Quarter": "2020 Q4", "Microsoft": 4174, "Google": 5479, "Meta": 4613, "Amazon": 14823}
+{"Quarter": "2021 Q1", "Microsoft": 5089, "Google": 5942, "Meta": 4303, "Amazon": 12082}
+{"Quarter": "2021 Q2", "Microsoft": 6452, "Google": 5496, "Meta": 4641, "Amazon": 14288}
+{"Quarter": "2021 Q3", "Microsoft": 5810, "Google": 6819, "Meta": 4346, "Amazon": 15748}
+{"Quarter": "2021 Q4", "Microsoft": 5865, "Google": 6383, "Meta": 5400, "Amazon": 18935}
+{"Quarter": "2022 Q1", "Microsoft": 5340, "Google": 9786, "Meta": 5441, "Amazon": 14951}
+{"Quarter": "2022 Q2", "Microsoft": 6871, "Google": 6828, "Meta": 7572, "Amazon": 15724}
+{"Quarter": "2022 Q3", "Microsoft": 6283, "Google": 7276, "Meta": 9375, "Amazon": 16378}
+{"Quarter": "2022 Q4", "Microsoft": 6274, "Google": 7595, "Meta": 9043, "Amazon": 16592}
+{"Quarter": "2023 Q1", "Microsoft": 6607, "Google": 6289, "Meta": 6823, "Amazon": 14207}
+{"Quarter": "2023 Q2", "Microsoft": 8943, "Google": 6888, "Meta": 6134, "Amazon": 11455}
+{"Quarter": "2023 Q3", "Microsoft": 9917, "Google": 8055, "Meta": 6543, "Amazon": 12479}
+{"Quarter": "2023 Q4", "Microsoft": 9735, "Google": 11019, "Meta": 7665, "Amazon": 14588}
+{"Quarter": "2024 Q1", "Microsoft": 10952, "Google": 12012, "Meta": 6400, "Amazon": 14925}
+{"Quarter": "2024 Q2", "Microsoft": 13873, "Google": 13186, "Meta": 8173, "Amazon": 17620}
+{"Quarter": "2024 Q3", "Microsoft": 14923, "Google": 13016, "Meta": 8258, "Amazon": 22620}
+{"Quarter": "2024 Q4", "Microsoft": 15804, "Google": 14276, "Meta": 14425, "Amazon": 27834}

bigcodebench_hard_average_leaderboard.jsonl

@@ -0,0 +1,62 @@
+{"model": "o1-2024-12-17", "score": 35.5}
+{"model": "gemini-exp-1206", "score": 34.1}
+{"model": "gpt-4-turbo-2024-04-09", "score": 32.1}
+{"model": "athene-v2-chat", "score": 32.1}
+{"model": "athene-v2-agent", "score": 31.4}
+{"model": "gpt-4o-2024-11-20", "score": 31.1}
+{"model": "gpt-4o-2024-08-06", "score": 30.8}
+{"model": "qwen2.5-coder-32b-instruct", "score": 30.8}
+{"model": "claude-3.5-sonnet-20241022", "score": 30.4}
+{"model": "claude-3.5-haiku-20241022", "score": 30.1}
+{"model": "claude-3.5-sonnet-20240620", "score": 29.4}
+{"model": "deepseek-coder-v2-instruct (2024-07-24)", "score": 29.4}
+{"model": "gemini-1.5-pro-exp-0827", "score": 29.4}
+{"model": "gemini-exp-1114", "score": 29.4}
+{"model": "o1-preview-2024-09-12 (temperature=1)", "score": 28.8}
+{"model": "deepseek-v2-chat (2024-06-28)", "score": 28.7}
+{"model": "llama-3.3-70b-instruct", "score": 28.4}
+{"model": "gemini-2.0-flash-exp", "score": 28.1}
+{"model": "gemini-1.5-pro-exp-0801", "score": 27.4}
+{"model": "o1-mini-2024-09-12 (temperature=1)", "score": 27.4}
+{"model": "gemini-exp-1121", "score": 27.4}
+{"model": "gemini-2.0-flash-thinking-exp-1219", "score": 27.4}
+{"model": "gpt-4o-2024-05-13", "score": 27.1}
+{"model": "deepseek-coder-v2-instruct", "score": 27}
+{"model": "gemini-1.5-pro-002", "score": 26.6}
+{"model": "grok-beta", "score": 26.6}
+{"model": "llama-3.1-405b-instruct", "score": 26.4}
+{"model": "deepseek-v2.5-1210", "score": 26.4}
+{"model": "deepseek-v2.5", "score": 26.1}
+{"model": "claude-3-opus-20240229", "score": 26}
+{"model": "mistral-large-instruct-2407", "score": 26}
+{"model": "gemini-1.5-pro-api-0514", "score": 25.4}
+{"model": "llama-3.1-70b-instruct", "score": 25.4}
+{"model": "qwen2.5-72b-instruct", "score": 25.4}
+{"model": "gpt-4o-mini-2024-07-18", "score": 25.3}
+{"model": "llama-3-70b-instruct", "score": 24.6}
+{"model": "qwen2.5-32b-instruct", "score": 24.6}
+{"model": "llama-3.1-nemotron-70b-instruct", "score": 24.6}
+{"model": "dracarys-llama-3.1-70b-instruct", "score": 24.3}
+{"model": "gemini-1.5-flash-api-0514", "score": 23.6}
+{"model": "llama-3-70b-synthia-v3.5", "score": 23.6}
+{"model": "claude-3-sonnet-20240229", "score": 23.4}
+{"model": "dracarys-72b-instruct", "score": 22.6}
+{"model": "hermes-2-theta-llama-3-70b", "score": 22.3}
+{"model": "phi-3.1-mini-128k-instruct", "score": 22}
+{"model": "hermes-2-pro-llama-3-70b", "score": 21.6}
+{"model": "gemini-1.5-flash-exp-0827", "score": 21.6}
+{"model": "qwen2.5-14b-instruct", "score": 20.9}
+{"model": "qwen2-72b-chat", "score": 20.6}
+{"model": "codestral-22b-v0.1", "score": 20.6}
+{"model": "qwen2.5-coder-7b-instruct", "score": 20.3}
+{"model": "gemma-2-27b-instruct", "score": 20}
+{"model": "gpt-3.5-turbo-0125", "score": 19.9}
+{"model": "mixtral-8x22b-instruct", "score": 19.9}
+{"model": "athene-70b", "score": 19.9}
+{"model": "deepseek-coder-33b-instruct", "score": 19.3}
+{"model": "whiterabbitneo-33b-v1.5", "score": 19.3}
+{"model": "reflectioncoder-ds-33b", "score": 18.9}
+{"model": "deepseek-v2-chat", "score": 18.6}
+{"model": "opencoder-8b-instruct", "score": 18.5}
+{"model": "claude-3-haiku-20240307", "score": 18.3}
+{"model": "gpt-4-0613", "score": 17.6}

codeforces_leaderboard.jsonl

@@ -1,4 +1,4 @@
-{"model": "o3", "score": 2727}
+{"model": "o3", "score": 2400}
 {"model": "o3-mini", "score": 2073}
 {"model": "o1", "score": 1673}
 {"model": "o1-mini", "score": 1650}

gaia_leaderboard.jsonl

@@ -0,0 +1,61 @@
+{"model": "h2oGPTe Agent v1.6.8 (claude-3-5-sonnet)", "score": 65.12}
+{"model": "Langfun Agent v2.0 (claude-3-5-sonnet, gemini-1.5-pro-002)", "score": 49.33}
+{"model": "barcelona v0.1 (claude sonnet 3.5)", "score": 46.18}
+{"model": "omne v0.1 (o1-preview, gpt-4o)", "score": 40.53}
+{"model": "Trase Agent v0.2 (fine-tuned gemini, gpt-4o, o1-preview)", "score": 39.53}
+{"model": "Multi Agent", "score": 38.87}
+{"model": "DynaSaur (gpt-4o)", "score": 38.21}
+{"model": "magentic-1 (o1)", "score": 38}
+{"model": "Trase Agent v0.1 (fine-tuned gpt-4o)", "score": 35.55}
+{"model": "sibyl system v0.2 (gpt-4o)", "score": 34.55}
+{"model": "HuggingFaceAgents (gpt-4o)", "score": 33.33}
+{"model": "tapeagent v0.2", "score": 33.22}
+{"model": "little_potato (yanzw gpt-4o)", "score": 32.89}
+{"model": "Multi-Agent Experiment v0.1 (gpt-4-turbo)", "score": 32.33}
+{"model": "magentic-1", "score": 32.33}
+{"model": "das_agent v0.2", "score": 32.33}
+{"model": "cola_v0.4", "score": 31.89}
+{"model": "modified hugging face agents + gpt-4o", "score": 31.23}
+{"model": "das_agent", "score": 31}
+{"model": "das_agent v0.3", "score": 30.33}
+{"model": "cola_v0.3", "score": 30.23}
+{"model": "gpt-4o-2024-08-06", "score": 29}
+{"model": "replicated hugging face agents + gpt-4o", "score": 29}
+{"model": "tapeagent v0.1", "score": 27.57}
+{"model": "das_agent v0.4 mini (fixed)", "score": 26.91}
+{"model": "sibyl system v0.2 (gpt-4o-2024-08-06)", "score": 26.58}
+{"model": "das_agent v0.4 mini", "score": 25.91}
+{"model": "mmac v1.1 (gpt4v gemini 1.5)", "score": 25.91}
+{"model": "modified sibyl system", "score": 25.91}
+{"model": "maac_v1", "score": 25.58}
+{"model": "uk ai safety institute internal (gpt-4-turbo)", "score": 25}
+{"model": "FRIDAY (gpt-4-turbo)", "score": 24.25}
+{"model": "cola_abl", "score": 23.26}
+{"model": "replicated hugging face agents + gpt-4o mini", "score": 22.67}
+{"model": "tapeagent v0.2 mini", "score": 21.93}
+{"model": "friday_without_learning (os-copilot gpt-4-turbo)", "score": 21.59}
+{"model": "ceylon", "score": 17.06}
+{"model": "tapeagent v0.1 mini", "score": 16.61}
+{"model": "dip (gpt-4-turbo)", "score": 15.95}
+{"model": "sibyl system v0.2 (gpt-4o-mini-2024-07-18)", "score": 15.61}
+{"model": "cola_v0.2", "score": 15.28}
+{"model": "chamomile", "score": 14.67}
+{"model": "clarity v1", "score": 14.05}
+{"model": "warm-up act (gpt-4-turbo)", "score": 12.96}
+{"model": "frc v5", "score": 12}
+{"model": "cola_v0.1", "score": 10.96}
+{"model": "somedayv1.2", "score": 10.3}
+{"model": "somedayv1.1", "score": 9.97}
+{"model": "frc v4", "score": 9.33}
+{"model": "stealth3", "score": 9.3}
+{"model": "stealth2", "score": 8.97}
+{"model": "someday1", "score": 8.97}
+{"model": "frc v3", "score": 8.67}
+{"model": "stealth", "score": 8.64}
+{"model": "gpt-4-turbo", "score": 6.67}
+{"model": "someday", "score": 6.31}
+{"model": "AutoGPT4 (gpt-4)", "score": 5}
+{"model": "gpt-4o-mini-2024-07-18", "score": 4.65}
+{"model": "gpt-4", "score": 4}
+{"model": "gpt-3.5-turbo", "score": 2.67}
+{"model": "alphaagent v0.1 (gpt-4o)", "score": 2.33}

gpqa_leaderboard.jsonl

@@ -0,0 +1,8 @@
+{"model": "o1-2024-12-17", "score": 76}
+{"model": "claude-3-5-sonnet-20240620", "score": 56}
+{"model": "gpt-4o-2024-05-13", "score": 49}
+{"model": "claude-3-opus-20240229", "score": 48}
+{"model": "gemini-1.5-pro-001", "score": 45}
+{"model": "gpt-4-1106-preview", "score": 43}
+{"model": "claude-2.0", "score": 35}
+{"model": "gpt-4-0613", "score": 33}

models.jsonl

@@ -1,39 +1,60 @@
 {"Name": "o3", "Release Date": "2024-12-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o3-mini", "Release Date": "2024-12-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-2024-12-17", "Release Date": "2024-12-17", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-2024-12-17 (temperature=1)", "Release Date": "2024-12-17", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-2.0-flash-exp", "Release Date": "2024-12-11", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "Jeremy Berman", "Release Date": "2024-12-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-exp-1206", "Release Date": "2024-12-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "llama-3.3-70b-instruct", "Release Date": "2024-12-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "claude-3-5-sonnet-20241022", "Release Date": "2024-10-22", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "claude-3.5-sonnet-20241022", "Release Date": "2024-10-22", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "claude-3-5-haiku-20241022", "Release Date": "2024-10-22", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "claude-3.5-haiku-20241022", "Release Date": "2024-10-22", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-1.5-pro-002", "Release Date": "2024-09-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1-preview-2024-09-12", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1-preview", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1-mini-2024-09-12", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1-mini", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "deepseek-v2.5", "Release Date": "2024-09-05", "Total Parameters": 236, "Active Parameters": 236, "API Cost": 0}
+{"Name": "deepseek-v2.5-0908", "Release Date": "2024-09-08", "Total Parameters": 236, "Active Parameters": 236, "API Cost": 0}
 {"Name": "qwen-plus-0828", "Release Date": "2024-08-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen-plus-0828", "Release Date": "2024-08-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-pro-exp-0827", "Release Date": "2024-08-27", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-flash-exp-0827", "Release Date": "2024-08-27", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-flash-8b-exp-0827", "Release Date": "2024-08-27", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "chatgpt-4o-latest", "Release Date": "2024-08-25", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "chatgpt-4o-latest-24-09-07", "Release Date": "2024-09-07", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "command-r-plus-08-2024", "Release Date": "2024-08-21", "Total Parameters": 104, "Active Parameters": 104, "API Cost": 0}
 {"Name": "command-r-08-2024", "Release Date": "2024-08-19", "Total Parameters": 32, "Active Parameters": 32, "API Cost": 0}
 {"Name": "gemini-1.5-pro-exp-0801", "Release Date": "2024-08-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "grok-2-1212", "Release Date": "2024-12-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "grok-2-2024-08-13", "Release Date": "2024-08-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-4o-2024-11-20", "Release Date": "2024-11-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-4o-2024-08-06", "Release Date": "2024-08-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4o", "Release Date": "2024-05-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4o-2024-05-13", "Release Date": "2024-05-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4o-mini-2024-07-18", "Release Date": "2024-07-18", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemma-2-9b-it-simpo", "Release Date": "2024-07-17", "Total Parameters": 9, "Active Parameters": 9, "API Cost": 0}
 {"Name": "claude-3-5-sonnet-20240620", "Release Date": "2024-06-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "grok-2-mini-2024-08-13", "Release Date": "2024-08-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "grok-beta", "Release Date": "2024-08-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-advanced-0514", "Release Date": "2024-05-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "llama-3.1-405b-instruct-bf16", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
 {"Name": "llama-3.1-405b-instruct-fp8", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
+{"Name": "llama-3.1-405b-instruct", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
 {"Name": "llama-3.1-405b", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
 {"Name": "gpt-4o-2024-08-06", "Release Date": "2024-08-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-pro-api-0514", "Release Date": "2024-05-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-pro-api-0409-preview", "Release Date": "2024-04-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4-turbo-2024-04-09", "Release Date": "2024-04-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4-1106-preview", "Release Date": "2023-11-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-4-turbo", "Release Date": "2023-11-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "mistral-large-2407", "Release Date": "2024-07-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "mistral-large-2", "Release Date": "2024-07-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "athene-70b-0725", "Release Date": "2024-07-25", "Total Parameters": 70, "Active Parameters": 70, "API Cost": 0}
 {"Name": "claude-3-opus-20240229", "Release Date": "2024-02-29", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "meta-llama-3.1-70b-instruct", "Release Date": "2024-07-23", "Total Parameters": 70, "Active Parameters": 70, "API Cost": 0}
 {"Name": "llama-3.1-70b-instruct", "Release Date": "2024-07-23", "Total Parameters": 70, "Active Parameters": 70, "API Cost": 0}
 {"Name": "gpt-4-0125-preview", "Release Date": "2024-01-25", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "yi-large-preview", "Release Date": "2024-06-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -47,6 +68,7 @@
 {"Name": "nemotron-4-340b-instruct", "Release Date": "2024-06-14", "Total Parameters": 340, "Active Parameters": 340, "API Cost": 0}
 {"Name": "bard-jan-24-gemini-pro", "Release Date": "2024-01-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-pro-001", "Release Date": "2024-02-15", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-1.5-pro", "Release Date": "2024-02-15", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "glm-4-0520", "Release Date": "2024-05-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0.63}
 {"Name": "llama-3-70b-instruct", "Release Date": "2024-04-18", "Total Parameters": 70, "Active Parameters": 70, "API Cost": 0}
 {"Name": "claude-3-sonnet-20240229", "Release Date": "2024-02-29", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -54,10 +76,10 @@
 {"Name": "reka-core-20240501", "Release Date": "2024-05-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "command-r-plus", "Release Date": "2024-04-04", "Total Parameters": 104, "Active Parameters": 104, "API Cost": 0}
 {"Name": "gemma-2-9b-it", "Release Date": "2024-06-27", "Total Parameters": 9, "Active Parameters": 9, "API Cost": 0}
+{"Name": "qwen2.5-coder-32b-instruct", "Release Date": "2024-09-18", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen2-72b-instruct", "Release Date": "2024-06-07", "Total Parameters": 72, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4-0314", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
-{"Name": "gpt-4-0314", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen-max-0428", "Release Date": "2024-04-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "glm-4-0116", "Release Date": "2024-01-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "claude-3-haiku-20240307", "Release Date": "2024-03-07", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -93,6 +115,7 @@
 {"Name": "gemini-pro", "Release Date": "2023-12-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen1.5-14b-chat", "Release Date": "2024-02-04", "Total Parameters": 14, "Active Parameters": 14, "API Cost": 0}
 {"Name": "gpt-3.5-turbo-0314", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-3.5-turbo", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "wizardlm-70b", "Release Date": "2023-08-09", "Total Parameters": 70, "Active Parameters": 70, "API Cost": 0}
 {"Name": "gpt-3.5-turbo-0125", "Release Date": "2024-01-25", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "dbrx-instruct-preview", "Release Date": "2024-03-27", "Total Parameters": 132, "Active Parameters": 36, "API Cost": 0}
@@ -153,4 +176,94 @@
 {"Name": "stablelm-tuned-alpha-7b", "Release Date": "2023-04-20", "Total Parameters": 7, "Active Parameters": 7, "API Cost": 0}
 {"Name": "dolly-v2-12b", "Release Date": "2023-04-12", "Total Parameters": 12, "Active Parameters": 12, "API Cost": 0}
 {"Name": "llama-13b", "Release Date": "2023-02-27", "Total Parameters": 13, "Active Parameters": 13, "API Cost": 0}
-{"Name": "gpt-3.5", "Release Date": "2022-11-30", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-3.5", "Release Date": "2022-11-30", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "deepseek-coder-v2-instruct (2024-07-24)", "Release Date": "2024-07-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-exp-1114", "Release Date": "2024-11-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "athene-v2-chat", "Release Date": "2024-11-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "athene-v2-agent", "Release Date": "2024-11-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "claude-3.5-sonnet-20240620", "Release Date": "2024-06-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-preview-2024-09-12 (temperature=1)", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "deepseek-v2-chat (2024-06-28)", "Release Date": "2024-06-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-mini-2024-09-12 (temperature=1)", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-exp-1121", "Release Date": "2024-11-21", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-2.0-flash-thinking-exp-1219", "Release Date": "2024-12-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "deepseek-coder-v2-instruct", "Release Date": "2024-06-17", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "deepseek-v2.5-1210", "Release Date": "2024-12-10", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "mistral-large-instruct-2407", "Release Date": "2024-07-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "qwen2.5-72b-instruct", "Release Date": "2024-09-15", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "qwen2.5-32b-instruct", "Release Date": "2024-11-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "llama-3.1-nemotron-70b-instruct", "Release Date": "2024-10-02", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "dracarys-llama-3.1-70b-instruct", "Release Date": "2024-08-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "llama-3-70b-synthia-v3.5", "Release Date": "2024-05-26", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "dracarys-72b-instruct", "Release Date": "2024-08-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "hermes-2-theta-llama-3-70b", "Release Date": "2024-06-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "phi-3.1-mini-128k-instruct", "Release Date": "2024-07-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "hermes-2-pro-llama-3-70b", "Release Date": "2024-06-25", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "qwen2.5-14b-instruct", "Release Date": "2024-09-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "qwen2-72b-chat", "Release Date": "2024-05-22", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "codestral-22b-v0.1", "Release Date": "2024-05-29", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "qwen2.5-coder-7b-instruct", "Release Date": "2024-09-17", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemma-2-27b-instruct", "Release Date": "2024-06-27", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "mixtral-8x22b-instruct", "Release Date": "2024-04-17", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "athene-70b", "Release Date": "2024-07-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "deepseek-coder-33b-instruct", "Release Date": "2023-11-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "whiterabbitneo-33b-v1.5", "Release Date": "2024-02-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "reflectioncoder-ds-33b", "Release Date": "2024-05-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "deepseek-v2-chat", "Release Date": "2024-04-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "opencoder-8b-instruct", "Release Date": "2024-11-07", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "h2oGPTe Agent v1.6.8 (claude-3-5-sonnet)", "Release Date": "2024-12-21", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "Langfun Agent v2.0 (claude-3-5-sonnet, gemini-1.5-pro-002)", "Release Date": "2024-12-05", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "barcelona v0.1 (claude sonnet 3.5)", "Release Date": "2024-12-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "omne v0.1 (o1-preview, gpt-4o)", "Release Date": "2024-10-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "Trase Agent v0.2 (fine-tuned gemini, gpt-4o, o1-preview)", "Release Date": "2024-10-11", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "Multi Agent", "Release Date": "2024-10-10", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "DynaSaur (gpt-4o)", "Release Date": "2024-10-04", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "magentic-1 (o1)", "Release Date": "2024-10-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "Trase Agent v0.1 (fine-tuned gpt-4o)", "Release Date": "2024-09-04", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "sibyl system v0.2 (gpt-4o)", "Release Date": "2024-11-03", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "HuggingFaceAgents (gpt-4o)", "Release Date": "2024-06-27", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "tapeagent v0.2", "Release Date": "2024-12-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "little_potato (yanzw gpt-4o)", "Release Date": "2024-10-15", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "Multi-Agent Experiment v0.1 (gpt-4-turbo)", "Release Date": "2024-03-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "magentic-1", "Release Date": "2024-10-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "das_agent v0.2", "Release Date": "2024-09-18", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "cola_v0.4", "Release Date": "2024-11-02", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "modified hugging face agents + gpt-4o", "Release Date": "2024-09-03", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "das_agent", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "das_agent v0.3", "Release Date": "2024-09-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "cola_v0.3", "Release Date": "2024-11-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "replicated hugging face agents + gpt-4o", "Release Date": "2024-09-11", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "tapeagent v0.1", "Release Date": "2024-10-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "das_agent v0.4 mini (fixed)", "Release Date": "2024-10-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "sibyl system v0.2 (gpt-4o-2024-08-06)", "Release Date": "2024-09-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "das_agent v0.4 mini", "Release Date": "2024-10-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "mmac v1.1 (gpt4v gemini 1.5)", "Release Date": "2024-04-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "modified sibyl system", "Release Date": "2024-08-29", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "maac_v1", "Release Date": "2024-04-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "uk ai safety institute internal (gpt-4-turbo)", "Release Date": "2024-04-21", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "FRIDAY (gpt-4-turbo)", "Release Date": "2024-01-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "cola_abl", "Release Date": "2024-12-18", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "replicated hugging face agents + gpt-4o mini", "Release Date": "2024-10-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "tapeagent v0.2 mini", "Release Date": "2024-12-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "friday_without_learning (os-copilot gpt-4-turbo)", "Release Date": "2024-01-30", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "ceylon", "Release Date": "2024-04-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "tapeagent v0.1 mini", "Release Date": "2024-10-15", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "dip (gpt-4-turbo)", "Release Date": "2024-04-02", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "sibyl system v0.2 (gpt-4o-mini-2024-07-18)", "Release Date": "2024-10-08", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "cola_v0.2", "Release Date": "2024-10-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "chamomile", "Release Date": "2024-03-05", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "clarity v1", "Release Date": "2024-02-27", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "warm-up act (gpt-4-turbo)", "Release Date": "2024-02-22", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "frc v5", "Release Date": "2024-04-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "cola_v0.1", "Release Date": "2024-10-23", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "somedayv1.2", "Release Date": "2024-06-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "somedayv1.1", "Release Date": "2024-06-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "frc v4", "Release Date": "2024-04-18", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "stealth3", "Release Date": "2024-02-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "stealth2", "Release Date": "2024-02-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "someday1", "Release Date": "2024-06-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "frc v3", "Release Date": "2024-04-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "stealth", "Release Date": "2024-02-19", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "someday", "Release Date": "2024-06-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "AutoGPT4 (gpt-4)", "Release Date": "2023-11-03", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "alphaagent v0.1 (gpt-4o)", "Release Date": "2024-10-24", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}

planbench_leaderboard.jsonl

@@ -1,3 +1,4 @@
 {"model": "o1-preview-2024-09-12", "score": 52.8}
 {"model": "llama-3.1-405b", "score": 0.8}
-{"model": "gpt-4", "score": 0.16}
+{"model": "gpt-4", "score": 0.16}
+{"model": "gpt-4o", "score": 0}

simple_bench_leaderboard.jsonl

@@ -1,10 +1,17 @@
-{"model": "claude-3-5-sonnet-20240620", "score": 27}
-{"model": "gpt-4-1106-preview", "score": 26}
-{"model": "claude-3-opus-20240229", "score": 25}
-{"model": "llama-3.1-405b-instruct-fp8", "score": 22}
-{"model": "gemini-1.5-pro-001", "score": 21}
-{"model": "gpt-4-0613", "score": 18}
-{"model": "gpt-4o-2024-05-13", "score": 16}
-{"model": "deepseek-v2-api-0628", "score": 15}
-{"model": "mistral-large-2407", "score": 13}
-{"model": "gpt-4o-mini-2024-07-18", "score": 5}
+{"model": "o1-preview-2024-09-12", "score": 41.7}
+{"model": "claude-3-5-sonnet-20241022", "score": 41.4}
+{"model": "o1-2024-12-17", "score": 36.7}
+{"model": "gemini-exp-1206", "score": 31.1}
+{"model": "claude-3-5-sonnet-20240620", "score": 27.5}
+{"model": "gemini-1.5-pro-002", "score": 27.1}
+{"model": "gpt-4-turbo-2024-04-09", "score": 25.1}
+{"model": "claude-3-opus-20240229", "score": 23.5}
+{"model": "llama-3.1-405b-instruct-fp8", "score": 23.0}
+{"model": "grok-beta", "score": 22.7}
+{"model": "mistral-large-2407", "score": 22.5}
+{"model": "llama-3.3-70b-instruct", "score": 19.9}
+{"model": "gemini-2.0-flash-exp", "score": 18.9}
+{"model": "o1-mini-2024-09-12", "score": 18.1}
+{"model": "gpt-4o-2024-08-06", "score": 17.8}
+{"model": "command-r-plus", "score": 17.4}
+{"model": "gpt-4o-mini-2024-07-18", "score": 10.7}

zeroeval_average_leaderboard.jsonl

@@ -0,0 +1,21 @@
+{"model": "o1-preview-2024-09-12", "score": 86.1}
+{"model": "o1-mini-2024-09-12", "score": 80.6}
+{"model": "claude-3-5-sonnet-20241022", "score": 67.1}
+{"model": "gemini-1.5-pro-exp-0827", "score": 66.1}
+{"model": "gpt-4o-2024-08-06", "score": 65.6}
+{"model": "chatgpt-4o-latest-24-09-07", "score": 64.6}
+{"model": "gpt-4o-2024-05-13", "score": 64.3}
+{"model": "claude-3-5-sonnet-20240620", "score": 63.0}
+{"model": "grok-2-1212", "score": 62.8}
+{"model": "qwen2.5-72b-instruct", "score": 61.6}
+{"model": "llama-3.1-405b-instruct", "score": 59.8}
+{"model": "gpt-4-turbo-2024-04-09", "score": 59.8}
+{"model": "gemini-1.5-flash-exp-0827", "score": 59.0}
+{"model": "mistral-large-2", "score": 58.9}
+{"model": "gpt-4o-mini-2024-07-18", "score": 57.4}
+{"model": "deepseek-v2.5-0908", "score": 54.3}
+{"model": "claude-3-opus-20240229", "score": 54.2}
+{"model": "meta-llama-3.1-70b-instruct", "score": 53.8}
+{"model": "claude-3-5-haiku-20241022", "score": 53.4}
+{"model": "gemini-1.5-pro", "score": 52.5}
+{"model": "gpt-4-0314", "score": 52.3}

zeroeval_crux_leaderboard.jsonl

@@ -0,0 +1,21 @@
+{"model": "o1-preview-2024-09-12", "score": 95.9}
+{"model": "o1-mini-2024-09-12", "score": 93.8}
+{"model": "claude-3-5-sonnet-20241022", "score": 83.9}
+{"model": "gemini-1.5-pro-exp-0827", "score": 79.6}
+{"model": "gpt-4o-2024-08-06", "score": 87.0}
+{"model": "chatgpt-4o-latest-24-09-07", "score": 86.5}
+{"model": "gpt-4o-2024-05-13", "score": 86.1}
+{"model": "claude-3-5-sonnet-20240620", "score": 80.8}
+{"model": "grok-2-1212", "score": 75.3}
+{"model": "qwen2.5-72b-instruct", "score": 73.9}
+{"model": "llama-3.1-405b-instruct", "score": 73.0}
+{"model": "gpt-4-turbo-2024-04-09", "score": 78.9}
+{"model": "gemini-1.5-flash-exp-0827", "score": 74.5}
+{"model": "mistral-large-2", "score": 75.1}
+{"model": "gpt-4o-mini-2024-07-18", "score": 75.9}
+{"model": "deepseek-v2.5-0908", "score": 70.0}
+{"model": "claude-3-opus-20240229", "score": 70.4}
+{"model": "meta-llama-3.1-70b-instruct", "score": 64.3}
+{"model": "claude-3-5-haiku-20241022", "score": 68.8}
+{"model": "gemini-1.5-pro", "score": 68.0}
+{"model": "gpt-4-0314", "score": 74.5}

zeroeval_math_l5_leaderboard.jsonl

@@ -0,0 +1,21 @@
+{"model": "o1-preview-2024-09-12", "score": 84.5}
+{"model": "o1-mini-2024-09-12", "score": 89.3}
+{"model": "claude-3-5-sonnet-20241022", "score": 59.4}
+{"model": "gemini-1.5-pro-exp-0827", "score": 68.1}
+{"model": "gpt-4o-2024-08-06", "score": 55.3}
+{"model": "chatgpt-4o-latest-24-09-07", "score": 53.1}
+{"model": "gpt-4o-2024-05-13", "score": 54.8}
+{"model": "claude-3-5-sonnet-20240620", "score": 51.9}
+{"model": "grok-2-1212", "score": 60.9}
+{"model": "qwen2.5-72b-instruct", "score": 60.2}
+{"model": "llama-3.1-405b-instruct", "score": 49.8}
+{"model": "gpt-4-turbo-2024-04-09", "score": 46.5}
+{"model": "gemini-1.5-flash-exp-0827", "score": 54.5}
+{"model": "mistral-large-2", "score": 48.5}
+{"model": "gpt-4o-mini-2024-07-18", "score": 52.2}
+{"model": "deepseek-v2.5-0908", "score": 44.7}
+{"model": "claude-3-opus-20240229", "score": 36.9}
+{"model": "meta-llama-3.1-70b-instruct", "score": 43.1}
+{"model": "claude-3-5-haiku-20241022", "score": 46.5}
+{"model": "gemini-1.5-pro", "score": 39.8}
+{"model": "gpt-4-0314", "score": 26.1}

zeroeval_mmlu_redux_leaderboard.jsonl

@@ -0,0 +1,21 @@
+{"model": "o1-preview-2024-09-12", "score": 92.8}
+{"model": "o1-mini-2024-09-12", "score": 86.7}
+{"model": "claude-3-5-sonnet-20241022", "score": 88.9}
+{"model": "gemini-1.5-pro-exp-0827", "score": 86.1}
+{"model": "gpt-4o-2024-08-06", "score": 88.3}
+{"model": "chatgpt-4o-latest-24-09-07", "score": 88.9}
+{"model": "gpt-4o-2024-05-13", "score": 88.0}
+{"model": "claude-3-5-sonnet-20240620", "score": 86.0}
+{"model": "grok-2-1212", "score": 87.4}
+{"model": "qwen2.5-72b-instruct", "score": 85.6}
+{"model": "llama-3.1-405b-instruct", "score": 86.2}
+{"model": "gpt-4-turbo-2024-04-09", "score": 85.3}
+{"model": "gemini-1.5-flash-exp-0827", "score": 82.1}
+{"model": "mistral-large-2", "score": 83.0}
+{"model": "gpt-4o-mini-2024-07-18", "score": 81.5}
+{"model": "deepseek-v2.5-0908", "score": 80.4}
+{"model": "claude-3-opus-20240229", "score": 82.5}
+{"model": "meta-llama-3.1-70b-instruct", "score": 83.0}
+{"model": "claude-3-5-haiku-20241022", "score": 79.6}
+{"model": "gemini-1.5-pro", "score": 82.8}
+{"model": "gpt-4-0314", "score": 81.6}

zeroeval_zebralogic_leaderboard.jsonl

@@ -0,0 +1,22 @@
+{"model": "o1-2024-12-17", "score": 81.0}
+{"model": "o1-preview-2024-09-12", "score": 71.4}
+{"model": "o1-mini-2024-09-12", "score": 52.6}
+{"model": "claude-3-5-sonnet-20241022", "score": 36.2}
+{"model": "gemini-1.5-pro-exp-0827", "score": 30.5}
+{"model": "gpt-4o-2024-08-06", "score": 31.7}
+{"model": "chatgpt-4o-latest-24-09-07", "score": 29.9}
+{"model": "gpt-4o-2024-05-13", "score": 28.2}
+{"model": "claude-3-5-sonnet-20240620", "score": 33.4}
+{"model": "grok-2-1212", "score": 27.7}
+{"model": "qwen2.5-72b-instruct", "score": 26.6}
+{"model": "llama-3.1-405b-instruct", "score": 30.1}
+{"model": "gpt-4-turbo-2024-04-09", "score": 28.4}
+{"model": "gemini-1.5-flash-exp-0827", "score": 25.0}
+{"model": "mistral-large-2", "score": 29.0}
+{"model": "gpt-4o-mini-2024-07-18", "score": 20.1}
+{"model": "deepseek-v2.5-0908", "score": 22.1}
+{"model": "claude-3-opus-20240229", "score": 27.0}
+{"model": "meta-llama-3.1-70b-instruct", "score": 24.9}
+{"model": "claude-3-5-haiku-20241022", "score": 18.7}
+{"model": "gemini-1.5-pro", "score": 19.4}
+{"model": "gpt-4-0314", "score": 27.1}