update layout

app.py

@@ -142,6 +142,11 @@ def filter_models(
     return filtered_df
 
 
+shown_columns = [
+    c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden
+]
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -150,56 +155,61 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
+                # with gr.Column():
+                # with gr.Row():
+                search_bar = gr.Textbox(
+                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                    show_label=False,
+                    elem_id="search-bar",
+                )
+
+                # with gr.Row():
+                #     shown_columns = gr.CheckboxGroup(
+                #         choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden],
+                #         value=[
+                #             c.name
+                #             for c in fields(AutoEvalColumn)
+                #             if c.displayed_by_default and not c.hidden and not c.never_hidden
+                #         ],
+                #         label="Select columns to show",
+                #         elem_id="column-select",
+                #         interactive=True,
+                #     )
+                # with gr.Row():
+                #     deleted_models_visibility = gr.Checkbox(
+                #         value=False, label="Show gated/private/deleted models", interactive=True
+                #     )
+                # with gr.Column(min_width=320):
+                # with gr.Box(elem_id="box-filter"):
+                filter_columns_type = gr.CheckboxGroup(
+                    label="Model types",
+                    choices=[t.to_str() for t in ModelType],
+                    value=[t.to_str() for t in ModelType],
+                    interactive=True,
+                    elem_id="filter-columns-type",
+                )
+                # filter_columns_precision = gr.CheckboxGroup(
+                #     label="Precision",
+                #     choices=[i.value.name for i in Precision],
+                #     value=[i.value.name for i in Precision],
+                #     interactive=True,
+                #     elem_id="filter-columns-precision",
+                # )
+                filter_columns_size = gr.CheckboxGroup(
+                    label="Model sizes (in billions of parameters)",
+                    choices=list(NUMERIC_INTERVALS.keys()),
+                    value=list(NUMERIC_INTERVALS.keys()),
+                    interactive=True,
+                    elem_id="filter-columns-size",
+                )
 
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                value=leaderboard_df[
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.displayed_by_default]
+                ],  # ,# ] + shown_columns],
+                headers=[
+                    c.name for c in fields(AutoEvalColumn) if c.never_hidden or c.displayed_by_default
+                ],  ##, if c.never_hidden] + shown_columns,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -217,31 +227,31 @@ with demo:
                 update_table,
                 [
                     hidden_leaderboard_table_for_search,
-                    shown_columns,
+                    # None,
                     filter_columns_type,
-                    filter_columns_precision,
+                    # filter_columns_precision,
                     filter_columns_size,
-                    deleted_models_visibility,
+                    # None,
                     search_bar,
                 ],
                 leaderboard_table,
             )
             for selector in [
-                shown_columns,
+                # shown_columns,
                 filter_columns_type,
-                filter_columns_precision,
+                # filter_columns_precision,
                 filter_columns_size,
-                deleted_models_visibility,
+                # deleted_models_visibility,
             ]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
-                        shown_columns,
+                        # None,
                         filter_columns_type,
-                        filter_columns_precision,
+                        # filter_columns_precision,
                         filter_columns_size,
-                        deleted_models_visibility,
+                        # None,
                         search_bar,
                     ],
                     leaderboard_table,

src/about.py

@@ -7,6 +7,7 @@ class Task:
     benchmark: str
     metric: str
     col_name: str
+    category: str
     higher_is_better: bool = True
     scale_by_100: bool = True
 
@@ -15,23 +16,24 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    task1 = Task("ami_2020_aggressiveness", "f1,none", "AMI 2020 Agg")
-    task2 = Task("ami_2020_misogyny", "f1,none", "AMI 2020 Miso")
-    task0 = Task("arc_challenge_ita", "acc_norm,none", "ARC-C")
-    task4 = Task("belebele_ita", "acc_norm,none", "Belebele")
-    task3 = Task("gente_rephrasing", "acc,none", "GeNTE Neutralizing")
-    task12 = Task("haspeede2_hs", "f1,none", "HaSpeeDe2 HS")
-    task13 = Task("haspeede2_stereo", "f1,none", "HaSpeeDe2 Stereo")
-    task5 = Task("hatecheck_ita", "f1,none", "HateCheck")
-    task6 = Task("honest_ita", "acc,none", "HONEST", higher_is_better=False)
-    task14 = Task("ironita_irony", "f1,none", "IronITA Irony")
-    task15 = Task("ironita_sarcasm", "f1,none", "IronITA Sarcasm")
-    task7 = Task("itacola", "mcc,none", "ItaCoLA", scale_by_100=False)
-    task8 = Task("news_sum", "bertscore,none", "News Sum")
-    task16 = Task("sentipolc", "f1,none", "SENTIPOLC")
-    task9 = Task("squad_it", "squad_f1,get-answer", "SQuAD it")
-    task10 = Task("truthfulqa_mc2_ita", "acc,none", "TruthfulQA")
-    task11 = Task("xcopa_it", "acc,none", "XCOPA")
+    task1 = Task("ami_2020_aggressiveness", "f1,none", "AMI 2020 Agg", "NLU")
+    task2 = Task("ami_2020_misogyny", "f1,none", "AMI 2020 Miso", "NLU")
+    task0 = Task("arc_challenge_ita", "acc_norm,none", "ARC-C", "CFK")
+    task4 = Task("belebele_ita", "acc_norm,none", "Belebele", "NLU")
+    task3 = Task("gente_rephrasing", "acc,none", "GeNTE Neutralizing", "BFS")
+    task12 = Task("haspeede2_hs", "f1,none", "HaSpeeDe2 HS", "BFS")
+    task13 = Task("haspeede2_stereo", "f1,none", "HaSpeeDe2 Stereo", "BFS")
+    task5 = Task("hatecheck_ita", "f1,none", "HateCheck", "BFS")
+    task6 = Task("honest_ita", "acc,none", "HONEST", "BFS", higher_is_better=False)
+    task14 = Task("ironita_irony", "f1,none", "IronITA Irony", "NLU")
+    task15 = Task("ironita_sarcasm", "f1,none", "IronITA Sarcasm", "NLU")
+    task7 = Task("itacola", "mcc,none", "ItaCoLA", "NLU", scale_by_100=False)
+    task8 = Task("news_sum", "bertscore,none", "News Sum", "NLU")
+    task16 = Task("sentipolc", "f1,none", "SENTIPOLC", "NLU")
+    task9 = Task("squad_it", "squad_f1,get-answer", "SQuAD it", "CFK")
+    task10 = Task("truthfulqa_mc2_ita", "acc,none", "TruthfulQA", "CFK")
+    task11 = Task("xcopa_it", "acc,none", "XCOPA", "CFK")
+    task17 = Task("hellaswag_ita", "acc_norm,none", "Hellaswag-it", "CFK")
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/display/utils.py

@@ -32,7 +32,10 @@ auto_eval_column_dict.append(["training_codebase", ColumnContent, ColumnContent(
 auto_eval_column_dict.append(["training_data", ColumnContent, ColumnContent("Data", "str", True, False)])
 
 # Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg ⬆️", "number", True)])
+auto_eval_column_dict.append(["average_NLU", ColumnContent, ColumnContent("Avg NLU", "number", True)])
+auto_eval_column_dict.append(["average_CFK", ColumnContent, ColumnContent("Avg CFK", "number", True)])
+auto_eval_column_dict.append(["average_BFS", ColumnContent, ColumnContent("Avg BFS", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 

src/leaderboard/read_evals.py

@@ -104,7 +104,7 @@ class EvalResult:
             if task.scale_by_100:
                 mean_acc *= 100.0
 
-            results[task.benchmark] = mean_acc
+            results[task.benchmark] = {"value": mean_acc, "category": task.category}
 
         # pdb.set_trace()
 
@@ -141,7 +141,17 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+
+        # compute one average score per category
+        def _get_score_category(category):
+            filtered_scores = [v["value"] for _, v in self.results.items() if v["category"] == category]
+            return sum(filtered_scores) / len(filtered_scores)
+
+        average_NLU = _get_score_category("NLU")
+        average_CFK = _get_score_category("CFK")
+        average_BFS = _get_score_category("BFS")
+        average = (average_NLU + average_CFK + average_BFS) / 3
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -150,6 +160,9 @@ class EvalResult:
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.average_NLU.name: average_NLU,
+            AutoEvalColumn.average_CFK.name: average_CFK,
+            AutoEvalColumn.average_BFS.name: average_BFS,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.params.name: self.num_params,