big update

app.py

@@ -1,296 +1,521 @@
-# some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main
-import json
 import os
-from datetime import datetime, timezone
-
+import logging
+import time
+import schedule
+import datetime
 import gradio as gr
-import pandas as pd
-import requests
-from huggingface_hub import HfApi
-
-from src.css_html import custom_css
-from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT_3, CITATION_BUTTON_TEXT, CITATION_BUTTON_LABEL
-from src.utils import (
+from threading import Thread
+import datasets
+from huggingface_hub import snapshot_download, WebhooksServer, WebhookPayload, RepoCard
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+from apscheduler.schedulers.background import BackgroundScheduler
+
+# Start ephemeral Spaces on PRs (see config in README.md)
+from gradio_space_ci.webhook import IS_EPHEMERAL_SPACE, SPACE_ID, configure_space_ci
+
+from src.display.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    # INTRODUCTION_TEXT,
+    TITLE,
+    ABOUT_TEXT,
+    SUBMISSION_TEXT_3,
+)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
     AutoEvalColumn,
     fields,
-    is_model_on_hub,
-    make_clickable_names,
-    plot_elo_mle,
-    plot_solve_rate,
-    styled_error,
-    styled_message,
+    EvalQueueColumn
 )
-from datasets import load_dataset
-TOKEN = os.environ.get("TOKEN", None)
-api = HfApi(TOKEN)
-df = load_dataset("bigcode/bigcodebench-results", split="train").to_pandas().sort_values(["complete", "instruct"], ascending=False)
-task_elo_mle_df = load_dataset("bigcode/bigcodebench-elo", split="task_no_tie").to_pandas()
-bench_elo_mle_df = load_dataset("bigcode/bigcodebench-elo", split="benchmark_tie").to_pandas()
-complete_solve_rate = load_dataset("bigcode/bigcodebench-solve-rate", split="complete").to_pandas()
-instruct_solve_rate = load_dataset("bigcode/bigcodebench-solve-rate", split="instruct").to_pandas()
-
-QUEUE_REPO = "bigcode/bigcodebench-requests"
-EVAL_REQUESTS_PATH = "eval-queue"
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [
-    c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
-]
-TYPES_LITE = [
-    c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden
-]
-
-
-def add_new_eval(
-    model: str,
-    revision: str,
-    model_type: str,
-):
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # check the model actually exists before adding the eval
-    if revision == "":
-        revision = "main"
-
-    model_on_hub, error = is_model_on_hub(model, revision)
-    if not model_on_hub:
-        return styled_error(f'Model "{model}" {error}')
-
-    print("adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "revision": revision,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type.split(" ")[1],
-    }
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request.json"
-    print(f"Saving eval request to {out_path}")
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    api.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # remove the local file
-    os.remove(out_path)
-
-    return styled_message("Your request has been submitted to the evaluation queue!\n")
-
-
-def select_columns(df, columns):
-    always_here_cols = [
-        AutoEvalColumn.model_type_symbol.name,
-        AutoEvalColumn.model.name,
-    ]
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns]
-    ]
-    return filtered_df
-
-
-def filter_types(df, leaderboard_table, query):
-    if query == "all":
-        return df[leaderboard_table.columns]
-    else:
-        query = query[0]
-    filtered_df = df[df["type"].str.contains(query, na=False)]
-    return filtered_df[leaderboard_table.columns]
-
-
-def filter_direct_complete(df, leaderboard_table, query):
-    if query == "all":
-        return df[leaderboard_table.columns]
+from src.envs import (
+    API,
+    EVAL_REQUESTS_PATH,
+    RESULT_REPO,
+    HARD_RESULT_REPO,
+    ELO_REPO,
+    HARD_ELO_REPO,
+    SOLVE_REPO,
+    HARD_SOLVE_REPO,
+    HF_TOKEN,
+    QUEUE_REPO,
+    REPO_ID,
+    VOTES_REPO,
+    VOTES_PATH,
+    HF_HOME,
+)
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.tools.plots import plot_elo_mle, plot_solve_rate
+# from src.voting.vote_system import VoteManager, run_scheduler
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+# Start ephemeral Spaces on PRs (see config in README.md)
+from gradio_space_ci.webhook import IS_EPHEMERAL_SPACE, SPACE_ID, configure_space_ci
+
+# Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
+# This controls whether a full initialization should be performed.
+DO_FULL_INIT = True # os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
+NEW_DATA_ON_LEADERBOARD = True
+LEADERBOARD_DF = None
+HARD_LEADERBOARD_DF = None
+ELO_TASK_DF = None
+ELO_BENCH_DF = None
+HARD_ELO_TASK_DF = None
+HARD_ELO_BENCH_DF = None
+COMPLETE_SOLVE_DF = None
+INSTRUCT_SOLVE_DF = None
+HARD_COMPLETE_SOLVE_DF = None
+HARD_INSTRUCT_SOLVE_DF = None
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
+
+
+def time_diff_wrapper(func):
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        diff = end_time - start_time
+        logging.info(f"Time taken for {func.__name__}: {diff} seconds")
+        return result
+
+    return wrapper
+
+
+@time_diff_wrapper
+def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, backoff_factor=1.5):
+    """Download dataset with exponential backoff retries."""
+    attempt = 0
+    while attempt < max_attempts:
+        try:
+            logging.info(f"Downloading {repo_id} to {local_dir}")
+            snapshot_download(
+                repo_id=repo_id,
+                local_dir=local_dir,
+                repo_type=repo_type,
+                tqdm_class=None,
+                etag_timeout=30,
+                max_workers=8,
+            )
+            logging.info("Download successful")
+            return
+        except Exception as e:
+            wait_time = backoff_factor**attempt
+            logging.error(f"Error downloading {repo_id}: {e}, retrying in {wait_time}s")
+            time.sleep(wait_time)
+            attempt += 1
+    raise Exception(f"Failed to download {repo_id} after {max_attempts} attempts")
+
+def get_latest_data_leaderboard(
+    leaderboard_initial_df = None,
+    hard_leaderboard_initial_df = None,
+    elo_task_df = None,
+    elo_bench_df = None,
+    hard_elo_task_df = None,
+    hard_elo_bench_df = None,
+    complete_solve_df = None,
+    instruct_solve_df = None,
+    hard_complete_solve_df = None,
+    hard_instruct_solve_df = None
+    ):
+    global NEW_DATA_ON_LEADERBOARD
+    global LEADERBOARD_DF
+    global HARD_LEADERBOARD_DF
+    global ELO_TASK_DF
+    global ELO_BENCH_DF
+    global HARD_ELO_TASK_DF
+    global HARD_ELO_BENCH_DF
+    global COMPLETE_SOLVE_DF
+    global INSTRUCT_SOLVE_DF
+    global HARD_COMPLETE_SOLVE_DF
+    global HARD_INSTRUCT_SOLVE_DF
+
+    if NEW_DATA_ON_LEADERBOARD:
+        print("Leaderboard updated at reload!")
+        leaderboard_dataset = datasets.load_dataset(
+            RESULT_REPO, 
+            "default", 
+            split="train", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        )
+        LEADERBOARD_DF = get_leaderboard_df(
+            leaderboard_dataset=leaderboard_dataset, 
+            cols=COLS,
+        )
+        hard_leaderboard_dataset = datasets.load_dataset(
+            HARD_RESULT_REPO, 
+            "default", 
+            split="train", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        )
+        hard_leaderboard_df = get_leaderboard_df(
+            leaderboard_dataset=hard_leaderboard_dataset, 
+            cols=COLS,
+        )
+        HARD_LEADERBOARD_DF = hard_leaderboard_df
+        
+        elo_task_df = datasets.load_dataset(
+            ELO_REPO,
+            "default", 
+            split="task_no_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        elo_bench_df = datasets.load_dataset(
+            ELO_REPO,
+            "default", 
+            split="benchmark_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        ELO_TASK_DF = elo_task_df
+        ELO_BENCH_DF = elo_bench_df
+        
+        hard_elo_task_df = datasets.load_dataset(
+            HARD_ELO_REPO,
+            "default", 
+            split="task_no_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        hard_elo_bench_df = datasets.load_dataset(
+            HARD_ELO_REPO,
+            "default", 
+            split="benchmark_tie", 
+            cache_dir=HF_HOME, 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            verification_mode="no_checks"
+        ).to_pandas()
+        HARD_ELO_TASK_DF = hard_elo_task_df
+        HARD_ELO_BENCH_DF = hard_elo_bench_df
+        
+        complete_solve_df = datasets.load_dataset(
+            SOLVE_REPO,
+            "default",
+            split="complete",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()
+        instruct_solve_df = datasets.load_dataset(
+            SOLVE_REPO,
+            "default",
+            split="instruct",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()
+        COMPLETE_SOLVE_DF = complete_solve_df
+        INSTRUCT_SOLVE_DF = instruct_solve_df
+        
+        hard_complete_solve_df = datasets.load_dataset(
+            HARD_SOLVE_REPO,
+            "default",
+            split="complete",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()
+        hard_instruct_solve_df = datasets.load_dataset(
+            HARD_SOLVE_REPO,
+            "default",
+            split="instruct",
+            cache_dir=HF_HOME,
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            verification_mode="no_checks"
+        ).to_pandas()        
+        HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
+        HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
+        
+        NEW_DATA_ON_LEADERBOARD = False
 
-    if query == "chat template":
-        return df[~df["direct_complete"]][leaderboard_table.columns]
     else:
-        return df[df["direct_complete"]][leaderboard_table.columns]
-
-
-def search_table(df, leaderboard_table, query):
-    filtered_df = df[(df["model"].str.contains("|".join(q.strip() for q in query.split("|")), case=False))]
-    return filtered_df[leaderboard_table.columns]
-
+        LEADERBOARD_DF = leaderboard_initial_df
+        HARD_LEADERBOARD_DF = hard_leaderboard_initial_df
+        ELO_TASK_DF = elo_task_df
+        ELO_BENCH_DF = elo_bench_df
+        HARD_ELO_TASK_DF = hard_elo_task_df
+        HARD_ELO_BENCH_DF = hard_elo_bench_df
+        COMPLETE_SOLVE_DF = complete_solve_df
+        INSTRUCT_SOLVE_DF = instruct_solve_df
+        HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
+        HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
+        
+    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+
+
+def init_space():
+    """Initializes the application space, loading only necessary data."""
+
+    # Always redownload the leaderboard DataFrame
+    global LEADERBOARD_DF
+    global HARD_LEADERBOARD_DF
+    global ELO_TASK_DF
+    global ELO_BENCH_DF
+    global HARD_ELO_TASK_DF
+    global HARD_ELO_BENCH_DF
+    global COMPLETE_SOLVE_DF
+    global INSTRUCT_SOLVE_DF
+    global HARD_COMPLETE_SOLVE_DF
+    global HARD_INSTRUCT_SOLVE_DF
+    
+    LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
+
+    # Evaluation queue DataFrame retrieval is independent of initialization detail level
+    # eval_queue_dfs = get_latest_data_queue()
+
+    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+
+# Initialize VoteManager
+# vote_manager = VoteManager(VOTES_PATH, EVAL_REQUESTS_PATH, VOTES_REPO)
+
+
+# Schedule the upload_votes method to run every 15 minutes
+# schedule.every(15).minutes.do(vote_manager.upload_votes)
+
+# Start the scheduler in a separate thread
+# scheduler_thread = Thread(target=run_scheduler, args=(vote_manager,), daemon=True)
+# scheduler_thread.start()
+
+# Calls the init_space function with the `full_init` parameter determined by the `do_full_init` variable.
+# This initializes various DataFrames used throughout the application, with the level of initialization detail controlled by the `do_full_init` flag.
+LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, \
+ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, \
+COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, \
+HARD_INSTRUCT_SOLVE_DF = init_space()
+
+
+# Data processing for plots now only on demand in the respective Gradio tab
+# def load_and_create_plots():
+#     plot_df = create_plot_df(create_scores_df(LEADERBOARD_DF))
+#     return plot_df
+
+# Function to check if a user is logged in
+def check_login(profile: gr.OAuthProfile | None) -> bool:
+    if profile is None:
+        return False
+    return True
+
+def init_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.dummy],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.type.name, type="checkboxgroup", label="Model Types"),
+            ColumnFilter(AutoEvalColumn.openness.name, type="checkboxgroup", label="Openness"),
+            ColumnFilter(AutoEvalColumn.size_range.name, type="dropdown", label="Model Size"),
+            ColumnFilter(AutoEvalColumn.moe.name, type="checkboxgroup", label="Model Architecture"),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+        )
 
-df = make_clickable_names(df)
 
-demo = gr.Blocks(css=custom_css)
-with demo:
-    with gr.Row():
-        gr.Markdown(
-            """<div style="text-align: center;"><h1> 🌸<span style='color: #A74E95;'>Big</span><span style='color: #C867B5;'>Code</span><span style='color: #DD71C8;'>Bench</span> Leaderboard🌸</h1></div>\
-            <br>\
-            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">⭐ Big Code Models Leaderboard</a>, we compare performance of LLMs on <a href="https://huggingface.co/datasets/bigcode/bigcodebench">BigCodeBench</a> benchmark.</p>
-            <p>To get started, please check out <a href="https://github.com/bigcode-project/bigcodebench">our GitHub repository</a>.</p>
-""",
-            elem_classes="markdown-text",
-        )
+def init_others(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Gradio DataFrame is empty or None.")
+    return gr.Dataframe(dataframe, visible=False)
 
+main_block = gr.Blocks(css=custom_css)
+with main_block as demo:
+    with gr.Row(elem_id="header-row"):
+        gr.HTML(TITLE)
+    
+    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")    
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.Column():
-            with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
-                with gr.TabItem("🔍 Evaluation Table", id=0):
-                    with gr.Column():
-                        with gr.Accordion("➡️ See All Columns", open=False):
-                            shown_columns = gr.CheckboxGroup(
-                                choices=[
-                                    c
-                                    for c in COLS
-                                    if c
-                                    not in [
-                                        AutoEvalColumn.dummy.name,
-                                        AutoEvalColumn.model.name,
-                                        AutoEvalColumn.model_type_symbol.name,
-                                    ]
-                                ],
-                                value=[
-                                    c
-                                    for c in COLS_LITE
-                                    if c
-                                    not in [
-                                        AutoEvalColumn.dummy.name,
-                                        AutoEvalColumn.model.name,
-                                        AutoEvalColumn.model_type_symbol.name,
-                                    ]
-                                ],
-                                label="",
-                                elem_id="column-select",
-                                interactive=True,
-                            )
-                        # with gr.Column(min_width=780):
-                        with gr.Row():
-                            search_bar = gr.Textbox(
-                                placeholder="🔍 Separate multiple queries with '|'",
-                                show_label=False,
-                                elem_id="search-bar",
-                            )
-                            filter_types_columns = gr.Radio(
-                                label="⏚ Filter model types",
-                                choices=["all", "🟢 base", "🔶 instruction-tuned"], #, "EXT external-evaluation"],
-                                value="all",
-                                elem_id="filter-columns",
-                            )
-                            filter_prompting_columns = gr.Radio(
-                                label="⏚ Filter prompting",
-                                choices=["all", "chat template", "direct complete"],
-                                value="all",
-                                elem_id="filter-direct-complete",
-                            )
-                    leaderboard_df = gr.components.Dataframe(
-                        value=df[
-                            [
-                                AutoEvalColumn.model_type_symbol.name,
-                                AutoEvalColumn.model.name,
-                            ]
-                            + shown_columns.value
-                        ],
-                        headers=[
-                            AutoEvalColumn.model_type_symbol.name,
-                            AutoEvalColumn.model.name,
-                        ]
-                        + shown_columns.value,
-                        datatype=TYPES,
-                        elem_id="leaderboard-table",
-                        interactive=False,
-                    )
-
-                    hidden_leaderboard_df = gr.components.Dataframe(
-                        value=df,
-                        headers=COLS,
-                        datatype=["str" for _ in range(len(COLS))],
-                        visible=False,
-                    )
-                    search_bar.submit(
-                        search_table,
-                        [hidden_leaderboard_df, leaderboard_df, search_bar],
-                        leaderboard_df,
-                    )
-                    filter_types_columns.change(
-                        filter_types,
-                        [hidden_leaderboard_df, leaderboard_df, filter_types_columns],
-                        leaderboard_df,
-                    )
-                    filter_prompting_columns.change(
-                        filter_direct_complete,
-                        [hidden_leaderboard_df, leaderboard_df, filter_prompting_columns],
-                        leaderboard_df,
-                    )
-                    shown_columns.change(
-                        select_columns,
-                        [hidden_leaderboard_df, shown_columns],
-                        leaderboard_df,
-                    )
-                    gr.Markdown(
-                        """
-                    **Notes:**
-                    - _Complete_ vs _Instruct_:
-                        - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This variant tests if the models are good at coding.
-                        - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This variant tests if the models are really capable enough to understand human intents to code.
-                    - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants.
-                    - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on `BigCodeBench-Complete`, which starts from 1000 and is boostrapped 500 times.
-                    - `size` is the amount of activated model weight during inference.
-                    - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
-                    - For more details check the 📝 About section.
-                    """,
-                        elem_classes="markdown-text",
-                    )
-
-                with gr.TabItem("📊 Elo Rating", id=1):
-                    with gr.Column():
-                        with gr.Group():
-                            gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
-                            task_elo_map = gr.Plot()
-                            demo.load(plot_elo_mle, [gr.Dataframe(task_elo_mle_df, visible=False)], task_elo_map)
-                        with gr.Group():
-                            gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
-                            model_elo_map = gr.Plot()
-                            demo.load(plot_elo_mle, [gr.Dataframe(bench_elo_mle_df, visible=False)], model_elo_map)
-                        
-                with gr.TabItem("🧩 Solve Rate", id=2):
-                    with gr.Column():
-                        complete_map = gr.Plot()
-                        demo.load(plot_solve_rate, [gr.Dataframe(complete_solve_rate, visible=False),
-                                                    gr.Textbox("Complete", visible=False),
-                                                    ], complete_map)
-                        instruct_map = gr.Plot()
-                        demo.load(plot_solve_rate, [gr.Dataframe(instruct_solve_rate, visible=False),
-                                                    gr.Textbox("Instruct", visible=False),
-                                                    ], instruct_map)
-                        
-                with gr.TabItem("📝 About", id=3):
-                    gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
-                with gr.TabItem("Submit/Request Results 🚀", id=4):
-                    gr.Markdown(SUBMISSION_TEXT_3)
+        with gr.Tab("💎 Hard Set") as hard_tabs:
+            with gr.TabItem("🏅 Benchmark", elem_id="llm-benchmark-tab-table", id="hard_bench"):
+                hard_leaderboard = init_leaderboard(HARD_LEADERBOARD_DF)
+                gr.Markdown(
+                    """
+                **Notes:**
+                - _Hard_ vs _Full_:
+                    - <u>Hard</u>: A subset of ~150 BigCodeBench tasks which is more user-facing and challenging.
+                    - <u>Full</u>: The full set of 1140 BigCodeBench tasks.
+                - _Complete_ vs _Instruct_:
+                    - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This split tests if the models are good at coding.
+                    - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This split tests if the models are really capable enough to understand human intents to code.
+                - `Complete` and `Instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark splits.
+                - `Average` is the average of `Complete` and `Instruct` when both are available.
+                - `Elo Rating` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the BigCodeBench-Complete split. The rating starts from 1000 and is bootstrapped 500 times.
+                - `#Act Params (B)` is the number of activated model parameters during inference.
+                - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
+                - For more details check the 📝 About section.
+                """,
+                    elem_classes="markdown-text",
+                )
+            
+            with gr.TabItem("📊 Elo Rating", id="hard_elo"):
+                with gr.Column():
+                    with gr.Group():
+                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
+                        hard_task_elo_map = gr.Plot()
+                        hard_elo_task_gr = init_others(HARD_ELO_TASK_DF)
+                        demo.load(plot_elo_mle, [hard_elo_task_gr],
+                                    hard_task_elo_map)
+                    with gr.Group():
+                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
+                        hard_bench_elo_map = gr.Plot()
+                        hard_elo_bench_gr = init_others(HARD_ELO_BENCH_DF)
+                        demo.load(plot_elo_mle, [hard_elo_bench_gr],
+                                    hard_bench_elo_map)
                     
-        with gr.Row():
-            with gr.Accordion("📙 Citation", open=False):
-                citation_button = gr.Textbox(
-                    value=CITATION_BUTTON_TEXT,
-                    label=CITATION_BUTTON_LABEL,
-                    lines=20,
-                    elem_id="citation-button",
-                    show_copy_button=True,
+            with gr.TabItem("🧩 Solve Rate", id="hard_solve"):
+                with gr.Column():
+                    hard_complete_map = gr.Plot()
+                    hard_complete_solve_gr = init_others(HARD_COMPLETE_SOLVE_DF)
+                    demo.load(plot_solve_rate, [hard_complete_solve_gr,
+                                                gr.Textbox("Complete", visible=False),
+                                                gr.Number(10, visible=False),
+                                                gr.Number(16, visible=False),
+                                                ], hard_complete_map)
+                    hard_instruct_map = gr.Plot()
+                    hard_instruct_solve_gr = init_others(HARD_INSTRUCT_SOLVE_DF)
+                    demo.load(plot_solve_rate, [hard_instruct_solve_gr,
+                                                gr.Textbox("Instruct", visible=False),
+                                                gr.Number(10, visible=False),
+                                                gr.Number(16, visible=False),
+                                                ], hard_instruct_map)
+        with gr.Tab("🎯 Full Set") as full_tabs:
+            with gr.TabItem("🏅 Benchmark", elem_id="llm-benchmark-tab-table", id="full_bench"):
+                leaderboard = init_leaderboard(LEADERBOARD_DF)
+                gr.Markdown(
+                    """
+                **Notes:**
+                - _Complete_ vs _Instruct_:
+                    - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This variant tests if the models are good at coding.
+                    - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This variant tests if the models are really capable enough to understand human intents to code.
+                - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants.
+                - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the BigCodeBench-Complete split. The rating starts from 1000 and is bootstrapped 500 times.
+                - `size` is the amount of activated model weight during inference.
+                - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
+                - For more details check the 📝 About section.
+                """,
+                    elem_classes="markdown-text",
                 )
+            
+            with gr.TabItem("📊 Elo Rating", id="full_elo"):
+                with gr.Column():
+                    with gr.Group():
+                        
+                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
+                        task_elo_map = gr.Plot()
+                        elo_task_gr = init_others(ELO_TASK_DF)
+                        demo.load(plot_elo_mle, [elo_task_gr], task_elo_map)
+                    with gr.Group():
+                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
+                        bench_elo_map = gr.Plot()
+                        elo_bench_gr = init_others(ELO_BENCH_DF)
+                        demo.load(plot_elo_mle, [elo_bench_gr], bench_elo_map)
+                    
+            with gr.TabItem("🧩 Solve Rate", id="full_solve"):
+                with gr.Column():
+                    complete_map = gr.Plot()
+                    complete_solve_gr = init_others(COMPLETE_SOLVE_DF)
+                    demo.load(plot_solve_rate, [complete_solve_gr,
+                                                gr.Textbox("Complete", visible=False),
+                                                ], complete_map)
+                    instruct_map = gr.Plot()
+                    instruct_solve_gr = init_others(INSTRUCT_SOLVE_DF)
+                    demo.load(plot_solve_rate, [instruct_solve_gr,
+                                                gr.Textbox("Instruct", visible=False),
+                                                ], instruct_map)
+
+        with gr.TabItem("📝 About", id=3):
+            gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
+        with gr.TabItem("Request 🚀", id=4):
+            gr.Markdown(SUBMISSION_TEXT_3)
+    
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+                    
+    main_block.load(fn=get_latest_data_leaderboard, inputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
+    # leaderboard.change(fn=get_latest_data_queue, inputs=None, outputs=[finished_eval_table, running_eval_table, pending_eval_table])
+    # pending_eval_table.change(fn=vote_manager.create_request_vote_df, inputs=[pending_eval_table], outputs=[pending_eval_table_votes])
+
+main_block.queue(default_concurrency_limit=40)
+
+
+def enable_space_ci_and_return_server(ui: gr.Blocks) -> WebhooksServer:
+    # Taken from https://huggingface.co/spaces/Wauplin/gradio-space-ci/blob/075119aee75ab5e7150bf0814eec91c83482e790/src/gradio_space_ci/webhook.py#L61
+    # Compared to original, this one do not monkeypatch Gradio which allows us to define more webhooks.
+    # ht to Lucain!
+    if SPACE_ID is None:
+        print("Not in a Space: Space CI disabled.")
+        return WebhooksServer(ui=main_block)
+
+    if IS_EPHEMERAL_SPACE:
+        print("In an ephemeral Space: Space CI disabled.")
+        return WebhooksServer(ui=main_block)
+
+    card = RepoCard.load(repo_id_or_path=SPACE_ID, repo_type="space")
+    config = card.data.get("space_ci", {})
+    print(f"Enabling Space CI with config from README: {config}")
+
+    return configure_space_ci(
+        blocks=ui,
+        trusted_authors=config.get("trusted_authors"),
+        private=config.get("private", "auto"),
+        variables=config.get("variables", "auto"),
+        secrets=config.get("secrets"),
+        hardware=config.get("hardware"),
+        storage=config.get("storage"),
+    )
 
-demo.launch()
+# Create webhooks server (with CI url if in Space and not ephemeral)
+webhooks_server = enable_space_ci_and_return_server(ui=main_block)
+
+# Add webhooks
+@webhooks_server.add_webhook
+def update_leaderboard(payload: WebhookPayload) -> None:
+    """Redownloads the leaderboard dataset each time it updates"""
+    if payload.repo.type == "dataset" and payload.event.action == "update":
+        global NEW_DATA_ON_LEADERBOARD
+        if NEW_DATA_ON_LEADERBOARD:
+            return
+        NEW_DATA_ON_LEADERBOARD = True
+
+        for repo in [RESULT_REPO, HARD_RESULT_REPO, ELO_REPO, HARD_ELO_REPO, SOLVE_REPO, HARD_SOLVE_REPO]:
+            datasets.load_dataset(
+                repo, 
+                "default", 
+                cache_dir=HF_HOME, 
+                download_mode=datasets.DownloadMode.FORCE_REDOWNLOAD, 
+                verification_mode="no_checks"
+            )
+        
+        
+
+webhooks_server.launch()
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
+scheduler.start()

requirements.txt

@@ -1,19 +1,23 @@
-APScheduler
-black
-click
-datasets
-gradio
-gradio_client
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
 huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-requests
-tqdm
-transformers
-tokenizers>=0.15.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
-accelerate
+matplotlib==3.8.4
+numpy==1.26.0
+pandas==2.2.2
+plotly==5.14.1
+python-dateutil==2.8.2
 sentencepiece
-plotly
+tqdm==4.65.0
+transformers==4.41.1
+tokenizers>=0.15.0
+gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/[email protected] # CI !!!
+isort
+ruff
+gradio==4.31.0
+gradio[oauth]
+gradio_leaderboard==0.0.11
+requests==2.31.0
+requests-oauthlib== 1.3.1
+schedule == 1.2.2

src/{text_content.py → display/about.py}

@@ -1,3 +1,11 @@
+TITLE = """<div style="text-align: center;"><h1> 🌸<span style='color: #C867B5;'>BigCodeBench</span> Leaderboard🌸</h1></div>\
+            <br>\
+            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard">⭐ Big Code Models Leaderboard</a>, we compare performance of LLMs on <a href="https://huggingface.co/datasets/bigcode/bigcodebench">BigCodeBench</a> benchmark.</p>
+            <p>To get started, please check out <a href="https://github.com/bigcode-project/bigcodebench">our GitHub repository</a>.
+            <br>\
+            For more details, please check our <a href="https://huggingface.co/blog/leaderboard-bigcodebench-hard">blog on the Hard Set</a>, <a href="https://huggingface.co/blog/leaderboard-bigcodebench">blog on the Full Set</a> and <a href="https://arxiv.org/abs/2406.15877">paper</a>.</p>
+            """
+
 ABOUT_TEXT = """# Context
 We believe that there are three main expectations of a good execution-based programming benchmark:
 1. The benchmark should be easy to use and efficient in evaluating the fundamental capabilities of LLMs. Repo-level and agent-centric benchmarks (e.g., SWE-bench) are not suitable for this purpose.
@@ -135,5 +143,6 @@ CITATION_BUTTON_TEXT = r"""
 """
 
 SUBMISSION_TEXT_3="""
-We welcome the community to request for new models to be added to the leaderboard. Please [submit an issue here](https://github.com/bigcode-project/bigcodebench/issues/new/choose) to add the model to the leaderboard 🤗
+## We welcome the community to request for new models to be added to the leaderboard.
+## Please [file an issue](https://github.com/bigcode-project/bigcodebench/issues/new/choose) to add the model to the leaderboard or [start a discussion](https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard/discussions/new) in the community🤗
 """

src/{css_html.py → display/css_html_js.py}

@@ -1,13 +1,18 @@
-# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/assets/css_html_js.py
 custom_css = """
-#changelog-text {
-    font-size: 16px !important;
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
 }
 
-#changelog-text h2 {
-    font-size: 18px !important;
+/* Full width space */
+.gradio-container {
+    max-width: 95% !important;
 }
 
+/* Text style and margins */
 .markdown-text {
     font-size: 16px !important;
 }
@@ -29,51 +34,82 @@ custom_css = """
     transform: scale(1.3);
 }
 
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
 #search-bar-table-box > div:first-child {
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }
 
-/* Hides the final AutoEvalColumn */
-#llm-benchmark-tab-table table td:last-child,
-#llm-benchmark-tab-table table th:last-child {
-    display: none;
+.tab-buttons button {
+    font-size: 20px;
 }
 
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
+/* Filters style */
+#filter_type {
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span {
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap {
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner {
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input {
+    width: 1px;
+}
+#filter-columns-type {
+    border: 0;
+    padding: 0.5;
+}
+#filter-columns-size {
+    border: 0;
+    padding: 0.5;
+}
+#box-filter > .form {
+    border: 0;
 }
 
-.tab-buttons button {
-    font-size: 20px;
+/* Header styles */
+#header-title {
+    text-align: left;
+    display: inline-block;
 }
 
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
+#header-row {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
 }
 
-#scale-logo .download {
-    display: none;
+#header-row .gradio-html {
+    flex-grow: 1;
 }
-"""
+
+#oauth-button {
+    height: auto;
+    min-width: max-content;
+    white-space: nowrap;
+    padding: 10px 20px;
+    border-radius: 4px;
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,37 @@
+from huggingface_hub import HfApi
+
+API = HfApi()
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(df, model_col, link_col):
+    df[model_col] = df.apply(
+        lambda row: model_hyperlink(row[link_col], row[model_col]), axis=1
+    )
+    df["Openness"] = df.apply(
+        lambda row: "Open" if "huggingface.co" in row[link_col] else "Closed", axis=1
+    )
+    return df
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,142 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+import json
+import logging
+from datetime import datetime
+import pandas as pd
+
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+
+# Convert ISO 8601 dates to datetime objects for comparison
+def parse_iso8601_datetime(date_str):
+    if date_str.endswith('Z'):
+        date_str = date_str[:-1] + '+00:00'
+    return datetime.fromisoformat(date_str)
+
+def parse_datetime(datetime_str):
+    formats = [
+        "%Y-%m-%dT%H-%M-%S.%f",  # Format with dashes
+        "%Y-%m-%dT%H:%M:%S.%f",  # Standard format with colons
+        "%Y-%m-%dT%H %M %S.%f",  # Spaces as separator
+    ]
+
+    for fmt in formats:
+        try:
+            return datetime.strptime(datetime_str, fmt)
+        except ValueError:
+            continue
+    # in rare cases set unix start time for files with incorrect time (legacy files)
+    logging.error(f"No valid date format found for: {datetime_str}")
+    return datetime(1970, 1, 1)
+
+
+def load_json_data(file_path):
+    """Safely load JSON data from a file."""
+    try:
+        with open(file_path, "r") as file:
+            return json.load(file)
+    except json.JSONDecodeError:
+        print(f"Error reading JSON from {file_path}")
+        return None  # Or raise an exception
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+column_map = {
+    "T": "T",
+    "model": "Model",
+    "type": "Model Type",
+    "size_range": "Size Range",
+    "complete": "Complete",
+    "instruct": "Instruct",
+    "average": "Average",
+    "elo_mle": "Elo Rating",
+    "link": "Link",
+    "act_param": "#Act Params (B)",
+    "size": "#Params (B)",
+    "moe": "MoE",
+    "lazy": "Lazy",
+    "openness": "Openness",
+    "direct_complete": "Direct Completion",
+}
+
+type_map = {
+    "🔶": "🔶 Chat Models (RLHF, DPO, IFT, ...)",
+    "🟢": "🟢 Base Models"
+}
+
+moe_map = {
+    True: "MoE",
+    False: "Dense"
+}
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass(frozen=True)
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+    dummy: bool = False
+
+
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["T", ColumnContent, ColumnContent(column_map["T"], "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent(column_map["model"], "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["type", ColumnContent, ColumnContent(column_map["type"], "str", False, True)])
+auto_eval_column_dict.append(["size_range", ColumnContent, ColumnContent(column_map["size_range"], "str", False, True)])
+# Scores
+auto_eval_column_dict.append(["complete", ColumnContent, ColumnContent(column_map["complete"], "number", True)])
+auto_eval_column_dict.append(["instruct", ColumnContent, ColumnContent(column_map["instruct"], "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(column_map["average"], "number", True)])
+auto_eval_column_dict.append(["elo_mle", ColumnContent, ColumnContent(column_map["elo_mle"], "number", True)])
+
+# Model information
+auto_eval_column_dict.append(["act_param", ColumnContent, ColumnContent(column_map["act_param"], "number", True)])
+auto_eval_column_dict.append(["link", ColumnContent, ColumnContent(column_map["link"], "str", False, True)])
+auto_eval_column_dict.append(["size", ColumnContent, ColumnContent(column_map["size"], "number", False)])
+auto_eval_column_dict.append(["lazy", ColumnContent, ColumnContent(column_map["lazy"], "bool", False, True)])
+auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent(column_map["moe"], "str", False, True)])
+auto_eval_column_dict.append(["openness", ColumnContent, ColumnContent(column_map["openness"], "str", False, True)])
+auto_eval_column_dict.append(["direct_complete", ColumnContent, ColumnContent(column_map["direct_complete"], "bool", False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model_link = ColumnContent("link", "markdown", True)
+    model_name = ColumnContent("model", "str", True)
+
+@dataclass
+class ModelDetails:
+    name: str
+    symbol: str = ""  # emoji, only for the model type
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+
+NUMERIC_INTERVALS = {
+    "?": pd.Interval(-1, 0, closed="right"),
+    "~1.5": pd.Interval(0, 2, closed="right"),
+    "~3": pd.Interval(2, 4, closed="right"),
+    "~7": pd.Interval(4, 9, closed="right"),
+    "~13": pd.Interval(9, 20, closed="right"),
+    "~35": pd.Interval(20, 45, closed="right"),
+    "~60": pd.Interval(45, 70, closed="right"),
+    "70+": pd.Interval(70, 10000, closed="right"),
+}

src/envs.py

@@ -0,0 +1,39 @@
+import os
+from huggingface_hub import HfApi
+
+# clone / pull the lmeh eval data
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+
+REPO_ID = "bigcode/bigcodebench-leaderboard"
+QUEUE_REPO = "bigcode/bigcodebench-requests"
+RESULT_REPO = "bigcode/bigcodebench-results"
+HARD_RESULT_REPO = "bigcode/bigcodebench-hard-results"
+
+ELO_REPO = "bigcode/bigcodebench-elo"
+HARD_ELO_REPO = "bigcode/bigcodebench-hard-elo"
+SOLVE_REPO = "bigcode/bigcodebench-solve-rate"
+HARD_SOLVE_REPO = "bigcode/bigcodebench-hard-solve-rate"
+
+VOTES_REPO = "bigcode/bigcodebench-votes"
+
+HF_HOME = os.getenv("HF_HOME", ".")
+
+# Check HF_HOME write access
+print(f"Initial HF_HOME set to: {HF_HOME}")
+
+if not os.access(HF_HOME, os.W_OK):
+    print(f"No write access to HF_HOME: {HF_HOME}. Resetting to current directory.")
+    HF_HOME = "."
+    os.environ["HF_HOME"] = HF_HOME
+else:
+    print("Write access confirmed for HF_HOME")
+
+VOTES_PATH = os.path.join(HF_HOME, "model-votes")
+EVAL_REQUESTS_PATH = os.path.join(HF_HOME, "eval-queue")
+
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
+HAS_HIGHER_RATE_LIMIT = []
+
+API = HfApi(token=HF_TOKEN)

src/populate.py

@@ -0,0 +1,50 @@
+import pathlib
+import pandas as pd
+from datasets import Dataset
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.display.utils import load_json_data, column_map, type_map, moe_map, NUMERIC_INTERVALS
+
+
+
+def get_evaluation_queue_df(save_path, cols):
+    """Generate dataframes for pending, running, and finished evaluation entries."""
+    save_path = pathlib.Path(save_path)
+    all_evals = []
+
+    for path in save_path.rglob("*.json"):
+        data = load_json_data(path)
+    # Organizing data by status
+    status_map = {
+        "PENDING": ["PENDING", "RERUN"],
+        "RUNNING": ["RUNNING"],
+        "FINISHED": ["FINISHED", "PENDING_NEW_EVAL"],
+    }
+    status_dfs = {status: [] for status in status_map}
+    for eval_data in all_evals:
+        for status, extra_statuses in status_map.items():
+            if eval_data["status"] in extra_statuses:
+                status_dfs[status].append(eval_data)
+
+    return tuple(pd.DataFrame(status_dfs[status], columns=cols) for status in ["FINISHED", "RUNNING", "PENDING"])
+
+
+def get_leaderboard_df(leaderboard_dataset: Dataset, cols: list):
+    """Retrieve and process leaderboard data."""
+    all_data_json = leaderboard_dataset.to_dict()
+    num_items = leaderboard_dataset.num_rows
+    all_data_json_list = [{k: all_data_json[k][ix] for k in all_data_json.keys()} for ix in range(num_items)]
+
+    df = pd.DataFrame.from_records(all_data_json_list)
+    # replace df.moe true to false, false to true
+    # map column names
+    df = df.rename(columns=column_map)
+    df[AutoEvalColumn.moe.name] = df[AutoEvalColumn.moe.name].map(moe_map)
+    df[AutoEvalColumn.T.name] = df[AutoEvalColumn.type.name]
+    df[AutoEvalColumn.type.name] = df[AutoEvalColumn.type.name].map(type_map)
+    df[AutoEvalColumn.average.name] = df.apply(lambda x: (x[AutoEvalColumn.complete.name] + x[AutoEvalColumn.instruct.name]) / 2 if not pd.isna(x[AutoEvalColumn.complete.name]) and not pd.isna(x[AutoEvalColumn.instruct.name]) else None, axis=1)
+    df[AutoEvalColumn.size_range.name] = df[AutoEvalColumn.size.name].apply(lambda x: next((k for k, v in NUMERIC_INTERVALS.items() if x in v), "?"))
+    df = make_clickable_model(df, AutoEvalColumn.model.name, AutoEvalColumn.link.name)
+    df = df.sort_values(by=[AutoEvalColumn.complete.name], ascending=False)
+    df = df[cols].round(decimals=2)
+    return df

src/{utils.py → tools/plots.py}

@@ -1,45 +1,6 @@
-# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
-from dataclasses import dataclass
 import plotly.graph_objects as go
-from transformers import AutoConfig
 import plotly.express as px
 import numpy as np
-# These classes are for user facing column names, to avoid having to change them
-# all around the code when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-
-
-def fields(raw_class):
-    return [
-        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
-    ]
-
-
-@dataclass(frozen=True)
-class AutoEvalColumn:  # Auto evals column
-    model_type_symbol = ColumnContent("type", "str", True)
-    model = ColumnContent("model", "markdown", True)
-    complete_score = ColumnContent("complete", "number", True)
-    instruct_score = ColumnContent("instruct", "number", True)
-    elo_mle = ColumnContent("elo_mle", "number", True)
-    dummy = ColumnContent("model", "str", True)
-    size = ColumnContent("size", "number", True)
-
-
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_names(df):
-    df["model"] = df.apply(
-        lambda row: model_hyperlink(row["link"], row["model"]), axis=1
-    )
-    return df
 
 
 def plot_elo_mle(df):
@@ -63,13 +24,6 @@ def plot_solve_rate(df, task, rows=30, cols=38):
     values = np.array(values)
         
     n = len(values)
-    if rows is None or cols is None:
-        cols = int(math.sqrt(n))
-        rows = cols if cols * cols >= n else cols + 1
-
-        while rows * cols < n:
-            cols += 1
-
     values = np.pad(values, (0, rows * cols - n), 'constant', constant_values=np.nan).reshape((rows, cols))
     keys = np.pad(keys, (0, rows * cols - n), 'constant', constant_values='').reshape((rows, cols))
 
@@ -102,40 +56,4 @@ def plot_solve_rate(df, task, rows=30, cols=38):
         # height=600,
     )
     
-    return fig
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)
-
-
-def is_model_on_hub(model_name: str, revision: str) -> bool:
-    try:
-        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
-        return True, None
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-        )
-
-    except Exception as e:
-        print(f"Could not get the model config from the hub.: {e}")
-        return False, "was not found on hub!"
+    return fig

src/voting/vote_system.py

@@ -0,0 +1,150 @@
+import json
+import logging
+import pathlib
+import pandas as pd
+import gradio as gr
+import schedule
+import time
+from datetime import datetime, timezone
+
+from src.envs import API
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class VoteManager:
+    def __init__(self, votes_path, eval_requests_path, repo_id):
+        self.votes_path = votes_path
+        self.eval_requests_path = eval_requests_path
+        self.repo_id = repo_id
+        self.vote_dataset = self.read_vote_dataset()
+        self.vote_check_set = self.make_check_set(self.vote_dataset)
+        self.votes_to_upload = []
+
+    def init_vote_dataset(self):
+        self.vote_dataset = self.read_vote_dataset()
+        self.vote_check_set = self.make_check_set(self.vote_dataset)
+
+    def read_vote_dataset(self):
+        result = []
+        votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
+        if votes_file.exists():
+            with open(votes_file, "r") as f:
+                for line in f:
+                    data = json.loads(line.strip())
+                    result.append(data)
+        result = pd.DataFrame(result)
+        return result
+
+    def make_check_set(self, vote_dataset: pd.DataFrame):
+        result = list()
+        for row in vote_dataset.itertuples(index=False, name='vote'):
+            result.append((row.model, row.revision, row.username))
+        return set(result)
+    
+    def get_model_revision(self, selected_model: str) -> str:
+        """Fetch the revision for the given model from the request files."""
+        for user_folder in pathlib.Path(self.eval_requests_path).iterdir():
+            if user_folder.is_dir():
+                for file in user_folder.glob("*.json"):
+                    with open(file, "r") as f:
+                        data = json.load(f)
+                        if data.get("model") == selected_model:
+                            return data.get("revision", "main")
+        return "main"
+
+    def create_request_vote_df(self, pending_models_df: gr.Dataframe):
+        if pending_models_df.empty or not "model_name" in pending_models_df.columns:
+            return pending_models_df
+        self.vote_dataset = self.read_vote_dataset()
+        vote_counts = self.vote_dataset.groupby(['model', 'revision']).size().reset_index(name='vote_count')
+
+        pending_models_df_votes = pd.merge(
+            pending_models_df, 
+            vote_counts, 
+            left_on=["model_name", 'revision'], 
+            right_on=['model', 'revision'], 
+            how='left'
+        )
+        # Filling empty votes
+        pending_models_df_votes['vote_count'] = pending_models_df_votes['vote_count'].fillna(0)
+        pending_models_df_votes = pending_models_df_votes.sort_values(by=["vote_count", "model_name"], ascending=[False, True])
+        # Removing useless columns
+        pending_models_df_votes = pending_models_df_votes.drop(["model_name", "model"], axis=1)
+        return pending_models_df_votes
+
+    # Function to be called when a user votes for a model
+    def add_vote(
+            self,
+            selected_model: str,
+            pending_models_df: gr.Dataframe,
+            profile: gr.OAuthProfile | None
+        ):
+        logger.debug(f"Type of list before usage: {type(list)}")
+        # model_name, revision, user_id, timestamp
+        if selected_model in ["str", ""]:
+            gr.Warning("No model selected")
+            return
+        
+        if profile is None:
+            gr.Warning("Hub Login required")
+            return
+
+        vote_username = profile.username
+        model_revision = self.get_model_revision(selected_model)
+        
+        # tuple (immutable) for checking than already voted for model
+        check_tuple = (selected_model, model_revision, vote_username)
+        if check_tuple in self.vote_check_set:
+            gr.Warning("Already voted for this model")
+            return
+        
+        current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        vote_obj = {
+            "model": selected_model,
+            "revision": model_revision,
+            "username": vote_username,
+            "timestamp": current_time
+        }
+
+        # Append the vote to the JSONL file
+        try:
+            votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
+            with open(votes_file, "a") as f:
+                f.write(json.dumps(vote_obj) + "\n")
+            logger.info(f"Vote added locally: {vote_obj}")
+
+            self.votes_to_upload.append(vote_obj)
+        except Exception as e:
+            logger.error(f"Failed to write vote to file: {e}")
+            gr.Warning("Failed to record vote. Please try again")
+            return
+        
+        self.vote_check_set.add(check_tuple)
+        gr.Info(f"Voted for {selected_model}")
+
+        return self.create_request_vote_df(pending_models_df)
+
+    def upload_votes(self):
+        if self.votes_to_upload:
+            votes_file = pathlib.Path(self.votes_path) / "votes_data.jsonl"
+            try:
+                with open(votes_file, "rb") as f:
+                    API.upload_file(
+                        path_or_fileobj=f,
+                        path_in_repo="votes_data.jsonl",
+                        repo_id=self.repo_id,
+                        repo_type="dataset",
+                        commit_message="Updating votes_data.jsonl with new votes",
+                    )
+                logger.info("Votes uploaded to votes repository")
+                self.votes_to_upload.clear()
+            except Exception as e:
+                logger.error(f"Failed to upload votes to repository: {e}")
+
+def run_scheduler(vote_manager):
+    while True:
+        schedule.run_pending()
+        time.sleep(1)