nathan-flagged-models-vis

app.py

@@ -78,9 +78,10 @@ def update_table(
     precision_query: str,
     size_query: list,
     show_deleted: bool,
+    show_flagged: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, show_flagged)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
@@ -128,7 +129,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, show_flagged: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -136,6 +137,9 @@ def filter_models(
     else:  # Show only still on the hub models
         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
+    if not show_flagged:
+        filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
+
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
@@ -147,6 +151,7 @@ def filter_models(
 
     return filtered_df
 
+leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False)
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -183,6 +188,9 @@ with demo:
                         deleted_models_visibility = gr.Checkbox(
                             value=False, label="Show private/deleted models", interactive=True
                         )
+                        flagged_models_visibility = gr.Checkbox(
+                            value=False, label="Show flagged models", interactive=True
+                        )
                 with gr.Column(min_width=320):
                     #with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
@@ -237,6 +245,7 @@ with demo:
                     filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    flagged_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
@@ -253,6 +262,7 @@ with demo:
                     filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    flagged_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
@@ -260,7 +270,7 @@ with demo:
             # Check query parameter once at startup and update search bar + hidden component
             demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
             
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, flagged_models_visibility]:
                 selector.change(
                     update_table,
                     [
@@ -270,6 +280,7 @@ with demo:
                         filter_columns_precision,
                         filter_columns_size,
                         deleted_models_visibility,
+                        flagged_models_visibility,
                         search_bar,
                     ],
                     leaderboard_table,

src/display/utils.py

@@ -51,6 +51,7 @@ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
@@ -80,6 +81,7 @@ baseline_row = {
     AutoEvalColumn.gsm8k.name: 0.21,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
+    AutoEvalColumn.flagged.name: False,
 }
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)

src/leaderboard/filter_models.py

@@ -14,6 +14,17 @@ FLAGGED_MODELS = {
     "AIDC-ai-business/Marcoroni-13B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
     "AIDC-ai-business/Marcoroni-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
     "fblgit/una-xaberius-34b-v1beta": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/444",
+    "jan-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "rwitz2/go-bruins-v2.1.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "rwitz2/go-bruins-v2.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/GreenNodeLM-v3olet-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/GreenNodeLM-7B-v4leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/LeoScorpius-GreenNode-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "viethq188/LeoScorpius-7B-Chat-DPO": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "GreenNode/GreenNodeLM-7B-v2leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "janai-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "ignos/LeoScorpius-GreenNode-Alpaca-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
+    "fblgit/una-cybertron-7b-v3-OMA": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
 }
 
 # Models which have been requested by orgs to not be submitted on the leaderboard
@@ -36,6 +47,9 @@ def flag_models(leaderboard_data: list[dict]):
             model_data[
                 AutoEvalColumn.model.name
             ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+            model_data[AutoEvalColumn.flagged.name] = True
+        else:
+            model_data[AutoEvalColumn.flagged.name] = False
 
 
 def remove_forbidden_models(leaderboard_data: list[dict]):