update

.gitattributes

@@ -271,3 +271,16 @@ results_qwen/Llama-3-70b-chat-hf.jpg filter=lfs diff=lfs merge=lfs -text
 results_qwen/gpt-4.csv filter=lfs diff=lfs merge=lfs -text
 results_qwen/gpt-4.jpg filter=lfs diff=lfs merge=lfs -text
 results_qwen/Llama-3-70b-chat-hf.pkl filter=lfs diff=lfs merge=lfs -text
+all_results.pkl filter=lfs diff=lfs merge=lfs -text
+results/Llama-3-70b-chat-hf.png filter=lfs diff=lfs merge=lfs -text
+results/dbrx-instruct.png filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-0613.png filter=lfs diff=lfs merge=lfs -text
+results/gpt-4-1106.png filter=lfs diff=lfs merge=lfs -text
+results/Llama-3-70b-chat-hf.jpg filter=lfs diff=lfs merge=lfs -text
+results/dbrx-instruct.jpg filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-0613.jpg filter=lfs diff=lfs merge=lfs -text
+results/gpt-4-1106.jpg filter=lfs diff=lfs merge=lfs -text
+results/gpt-4-1106.pkl filter=lfs diff=lfs merge=lfs -text
+results/Llama-3-70b-chat-hf.pkl filter=lfs diff=lfs merge=lfs -text
+results/dbrx-instruct.pkl filter=lfs diff=lfs merge=lfs -text
+results/gpt-3.5-0613.pkl filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,98 +1,49 @@
-import gradio as gr
-import pandas as pd
-from glob import glob
-import matplotlib.pyplot as plt
-import seaborn as sns
-from matplotlib.colors import ListedColormap, BoundaryNorm
-from glob import glob
 import os
+from glob import glob
 
-
+import gradio as gr
 import matplotlib.pyplot as plt
-import seaborn as sns
-from matplotlib.colors import ListedColormap, BoundaryNorm
 import pandas as pd
+import seaborn as sns
+from matplotlib.colors import BoundaryNorm, ListedColormap
+
+all_results = pd.read_pickle("all_results.pkl")
+
+
+def get_accuracy_dataframe(df):
+    # Calculate overall model accuracy
+    df['parsed_judge_response'] = df['parsed_judge_response'].astype(float)
+    model_accuracy = df.groupby('model_name')['parsed_judge_response'].mean().reset_index()
+    
+    # Calculate model accuracy per difficulty level
+    df['difficulty_level'] = df['difficulty_level'].astype(int)
+    model_accuracy_per_level = df.groupby(['model_name', 'difficulty_level'])['parsed_judge_response'].mean().reset_index()
+    model_accuracy_per_level_df = model_accuracy_per_level.pivot(index='model_name', columns='difficulty_level', values='parsed_judge_response')
+    
+    # Merge overall accuracy and level-based accuracy into a single DataFrame
+    model_accuracy_df = model_accuracy.merge(model_accuracy_per_level_df, on='model_name')
+    model_accuracy_df.rename(columns={1: 'level_1', 2: 'level_2', 3: 'level_3', 4: 'level_4', 5: 'level_5'}, inplace=True)
+    model_accuracy_df.rename(columns={'parsed_judge_response': 'Accuracy'}, inplace=True)
+    
+    # Multiply by 100 and format to one decimal point
+    model_accuracy_df = model_accuracy_df.applymap(lambda x: round(x * 100, 1) if isinstance(x, float) else x)
+    
+    # Add headers with icons
+    model_accuracy_df.columns = [
+        "🤖 Model Name",
+        "⭐ Overall",
+        "📈 Level 1",
+        "🔍 Level 2",
+        "📘 Level 3",
+        "🔬 Level 4",
+    ]
 
-
-# Load text benchmark results
-noncot_results = glob("results/*.pkl")
-noncot_results_qwen = glob("results_qwen/*.pkl")
-# Load vision benchmark results
-vision_results = glob("results-vision/*.pkl")
-# Load CoT text benchmark results
-cot_text_results = glob("results-cot/*.pkl")
-# Load CoT vision benchmark results
-# cot_vision_results = glob("results-vision-CoT/*.pkl")
-
-
-# Function to load data, add model type and name
-def load_data(files, model_type):
-    data = []
-    for file in files:
-        df = pd.read_pickle(file)
-        df["Model Type"] = model_type
-        df["Model Name"] = file.split("/")[-1].replace(".pkl", "")
-        data.append(df)
-    return pd.concat(data, ignore_index=True)
-
-
-# Load and label all data
-data = load_data(noncot_results, "Text Only")
-data_qwen = load_data(noncot_results_qwen, "Text Only")
-vision_data = load_data(vision_results, "Vision")
-cot_text_data = load_data(cot_text_results, "CoT Text Only")
-# cot_vision_data = load_data(cot_vision_results, "CoT Vision")
-
-# Combine all data into a single DataFrame
-all_data = pd.concat([data_qwen, vision_data, cot_text_data], ignore_index=True)
-
-all_model_names = all_data["Model Name"].unique()
-all_text_only_model_names = list(
-    all_data[all_data["Model Type"] == "Text Only"]["Model Name"].unique()
-)
-all_cot_text_only_models = list(
-    all_data[all_data["Model Type"] == "CoT Text Only"]["Model Name"].unique()
-)
-
-
-text_only_filtered_raw = None
-text_only_filtered_raw_cot = None
-
-## Continue with the cold code --
-# TODO: Update me to read from all_data for later
-
-
-# Load the csv files into a dict with keys being name of the file and values being the data
-data = {file: pd.read_pickle(file) for file in noncot_results}
-# Load the vision files into a dict
-vision_data = {file: pd.read_pickle(file) for file in vision_results}
-# Load the CoT text files into a dict
-cot_text_data = {file: pd.read_pickle(file) for file in cot_text_results}
-# Load the CoT vision files into a dict
-# cot_vision_data = {file: pd.read_pickle(file) for file in cot_vision_results}
-
-data_qwen = {file: pd.read_pickle(file) for file in noncot_results_qwen}
-
-
-intersection_df = pd.read_pickle(
-    "./intersection_results/gpt-3.5-judge-by_Qwen_5times_intersection_subset_1.pkl"
-)
-# accuracy for each model
-intersection_df_acc = (
-    intersection_df.groupby("model_name")["parsed_judge_response"].mean().reset_index()
-)
-intersection_df_acc["Accuracy"] = intersection_df_acc["parsed_judge_response"] * 100
-intersection_df_acc.drop("parsed_judge_response", axis=1, inplace=True)
-intersection_df_acc.sort_values("Accuracy", ascending=False, inplace=True)
-
-
-def calculate_accuracy(df):
-    return df["parsed_judge_response"].mean() * 100
+    model_accuracy_df.sort_values(by="⭐ Overall", ascending=False, inplace=True)
+    
+    return model_accuracy_df
 
 
-def accuracy_breakdown(df):
-    # 4 level accuracy
-    return (df.groupby("difficulty_level")["parsed_judge_response"].mean() * 100).values
+accuracy_df = get_accuracy_dataframe(all_results)
 
 
 # Define the column names with icons
@@ -114,450 +65,126 @@ column_names = [
     "Level 4 Accuracy",
 ]
 
-
-# Function to process data
-def process_data(data):
-    data_for_df = []
-    for file, df in data.items():
-        overall_accuracy = round(calculate_accuracy(df), 2)
-        breakdown_accuracy = [round(acc, 2) for acc in accuracy_breakdown(df)]
-        model_name = file.split("/")[-1].replace(".pkl", "")
-        data_for_df.append([model_name, overall_accuracy] + breakdown_accuracy)
-    return data_for_df
-
-
-# Process all data
-text_data_for_df = process_data(data)
-text_data_for_df_qwen = process_data(data_qwen)
-
-vision_data_for_df = process_data(vision_data)
-cot_text_data_for_df = process_data(cot_text_data)
-# cot_vision_data_for_df = process_data(cot_vision_data)
-
-# Create DataFrames
-accuracy_df = pd.DataFrame(text_data_for_df, columns=column_names)
-accuracy_df_qwen = pd.DataFrame(text_data_for_df_qwen, columns=column_names)
-vision_accuracy_df = pd.DataFrame(vision_data_for_df, columns=column_names)
-cot_text_accuracy_df = pd.DataFrame(cot_text_data_for_df, columns=column_names)
-# cot_vision_accuracy_df = pd.DataFrame(cot_vision_data_for_df, columns=column_names)
-
-
-# Function to finalize DataFrame
-def finalize_df(df):
-    df = df.round(1)  # Round to one decimal place
-    df = df.applymap(lambda x: f"{x:.1f}" if isinstance(x, (int, float)) else x)
-    df.columns = headers_with_icons
-    df.sort_values(by="⭐ Overall", ascending=False, inplace=True)
-    # add a new column with the order (index)
-    df["#"] = range(1, len(df) + 1)
-    # bring rank to the first column
-    cols = df.columns.tolist()
-    cols = cols[-1:] + cols[:-1]
-    df = df[cols]
-
-    return df
-
-
-# Finalize all DataFrames
-accuracy_df = finalize_df(accuracy_df)
-accuracy_df_qwen = finalize_df(accuracy_df_qwen)
-vision_accuracy_df = finalize_df(vision_accuracy_df)
-cot_text_accuracy_df = finalize_df(cot_text_accuracy_df)
-# cot_vision_accuracy_df = finalize_df(cot_vision_accuracy_df)
-
-
 def load_heatmap(evt: gr.SelectData):
     heatmap_image = gr.Image(f"results/{evt.value}.jpg")
     return heatmap_image
 
 
-def load_heatmap_qwen(evt: gr.SelectData):
-    heatmap_image = gr.Image(f"results_qwen/{evt.value}.jpg")
-    return heatmap_image
 
+# # Function to process data
+# def process_data(data):
+#     data_for_df = []
+#     for file, df in data.items():
+#         overall_accuracy = round(calculate_accuracy(df), 2)
+#         breakdown_accuracy = [round(acc, 2) for acc in accuracy_breakdown(df)]
+#         model_name = file.split("/")[-1].replace(".pkl", "")
+#         data_for_df.append([model_name, overall_accuracy] + breakdown_accuracy)
+#     return data_for_df
 
-def load_vision_heatmap(evt: gr.SelectData):
-    heatmap_image = gr.Image(f"results-vision/{evt.value}.jpg")
-    return heatmap_image
 
+# # Function to finalize DataFrame
+# def finalize_df(df):
+#     df = df.round(1)  # Round to one decimal place
+#     df = df.applymap(lambda x: f"{x:.1f}" if isinstance(x, (int, float)) else x)
+#     df.columns = headers_with_icons
+#     df.sort_values(by="⭐ Overall", ascending=False, inplace=True)
+#     # add a new column with the order (index)
+#     df["#"] = range(1, len(df) + 1)
+#     # bring rank to the first column
+#     cols = df.columns.tolist()
+#     cols = cols[-1:] + cols[:-1]
+#     df = df[cols]
 
-def load_cot_heatmap(evt: gr.SelectData):
-    heatmap_image = gr.Image(f"results-cot/{evt.value}.jpg")
-    return heatmap_image
+#     return df
 
 
-def load_cot_vision_heatmap(evt: gr.SelectData):
-    heatmap_image = gr.Image(f"results-vision-CoT/{evt.value}.jpg")
+def load_heatmap(evt: gr.SelectData):
+    heatmap_image = gr.Image(f"results/{evt.value}.jpg")
     return heatmap_image
 
 
-def calculate_order_by_first_substring(selected_models):
-    global text_only_filtered_raw
-    first_columns = all_data[all_data["substring_index"] == 1]
-    query_ids_df = first_columns[first_columns["Model Type"] == "Text Only"]
-    query_ids_df = query_ids_df[query_ids_df["Model Name"].isin(selected_models)]
-
-    query_ids_df = query_ids_df.groupby("query_id").filter(
-        lambda x: x["parsed_judge_response"].eq(1).all()
-    )
-
-    fsm_ids = query_ids_df.fsm_id.unique()
-
-    text_only = all_data[all_data["Model Type"] == "Text Only"]
-    text_only_filtered = text_only[text_only["fsm_id"].isin(fsm_ids)]
-    text_only_filtered_raw = text_only_filtered.copy()
-
-    query_ids = text_only_filtered.query_id.unique()
-    text_only_filtered = (
-        text_only_filtered.groupby(["Model Name"])["parsed_judge_response"]
-        .mean()
-        .reset_index()
-    )
-
-    text_only_filtered["Accuracy"] = text_only_filtered["parsed_judge_response"] * 100
-    text_only_filtered.drop("parsed_judge_response", axis=1, inplace=True)
-
-    text_only_filtered["Accuracy"] = text_only_filtered["Accuracy"].apply(
-        lambda x: round(x, 2)
-    )
-    text_only_filtered.sort_values("Accuracy", ascending=False, inplace=True)
-
-    number_of_queries = len(query_ids)
-    number_of_fsms = len(fsm_ids)
-
-    return text_only_filtered, number_of_queries, number_of_fsms
-
-
-def calculate_order_by_first_substring_cot(selected_models):
-    global text_only_filtered_raw_cot
-    first_columns = all_data[all_data["substring_index"] == 1]
-    query_ids_df = first_columns[first_columns["Model Type"] == "CoT Text Only"]
-    query_ids_df = query_ids_df[query_ids_df["Model Name"].isin(selected_models)]
-
-    query_ids_df = query_ids_df.groupby("query_id").filter(
-        lambda x: x["parsed_judge_response"].eq(1).all()
-    )
-
-    fsm_ids = query_ids_df.fsm_id.unique()
-
-    text_only = all_data[all_data["Model Type"] == "CoT Text Only"]
-    text_only_filtered = text_only[text_only["fsm_id"].isin(fsm_ids)]
-    text_only_filtered_raw_cot = text_only_filtered.copy()
-
-    query_ids = text_only_filtered.query_id.unique()
-    text_only_filtered = (
-        text_only_filtered.groupby(["Model Name"])["parsed_judge_response"]
-        .mean()
-        .reset_index()
-    )
-
-    text_only_filtered["Accuracy"] = text_only_filtered["parsed_judge_response"] * 100
-    text_only_filtered.drop("parsed_judge_response", axis=1, inplace=True)
-
-    text_only_filtered["Accuracy"] = text_only_filtered["Accuracy"].apply(
-        lambda x: round(x, 2)
-    )
-    text_only_filtered.sort_values("Accuracy", ascending=False, inplace=True)
-
-    number_of_queries = len(query_ids)
-    number_of_fsms = len(fsm_ids)
-
-    return text_only_filtered, number_of_queries, number_of_fsms
-
-
-def generate_heatmap_for_specific_model(model_name):
-    global text_only_filtered_raw
-
-    cmap = ListedColormap(["lightblue", "red", "green"])
-    bounds = [-1.5, -0.5, 0.5, 1.5]
-    norm = BoundaryNorm(bounds, cmap.N)
-
-    model_df = text_only_filtered_raw[
-        text_only_filtered_raw["Model Name"] == model_name
-    ]
-    model_df["fsm_info"] = model_df.apply(
-        lambda x: f"{x['num_states']} states, {x['num_alphabet']} alphabet", axis=1
-    )
-    model_df = model_df.sort_values(by=["num_states", "num_alphabet"])
-
-    pivot_df = (
-        model_df.pivot_table(
-            index="fsm_info",
-            columns="substring_index",
-            values="parsed_judge_response",
-            aggfunc="first",
-        )
-        .fillna(-1)
-        .astype(float)
-    )
-
-    # Dynamically adjust figure size
-    num_rows, num_cols = pivot_df.shape
-    fig_width = max(12, num_cols * 0.5)  # Adjust width per column
-    fig_height = max(8, num_rows * 0.4)  # Adjust height per row
-
-    fig, ax = plt.subplots(figsize=(fig_width, fig_height))
-    sns.heatmap(
-        pivot_df,
-        cmap=cmap,
-        linewidths=1,
-        linecolor="black",
-        norm=norm,
-        cbar=False,
-        square=True,
-        ax=ax,
-    )
-    plt.title(f"Heatmap for Model: {model_name}", fontsize=12)
-    plt.xlabel("Substring Index")
-    plt.ylabel("FSM (States, Alphabet)")
-    plt.xticks(rotation=45)
-
-    sns.despine(ax=ax, top=True, right=True, left=True, bottom=True)
-
-    return fig
-
-
-def generate_heatmap_for_specific_model_cot(model_name):
-    global text_only_filtered_raw_cot
-
-    cmap = ListedColormap(["lightblue", "red", "green"])
-    bounds = [-1.5, -0.5, 0.5, 1.5]
-    norm = BoundaryNorm(bounds, cmap.N)
-
-    model_df = text_only_filtered_raw_cot[
-        text_only_filtered_raw_cot["Model Name"] == model_name
-    ]
-    model_df["fsm_info"] = model_df.apply(
-        lambda x: f"{x['num_states']} states, {x['num_alphabet']} alphabet", axis=1
-    )
-    model_df = model_df.sort_values(by=["num_states", "num_alphabet"])
-
-    pivot_df = (
-        model_df.pivot_table(
-            index="fsm_info",
-            columns="substring_index",
-            values="parsed_judge_response",
-            aggfunc="first",
-        )
-        .fillna(-1)
-        .astype(float)
-    )
-
-    # Dynamically adjust figure size
-    num_rows, num_cols = pivot_df.shape
-    fig_width = max(12, num_cols * 0.5)  # Adjust width per column
-    fig_height = max(8, num_rows * 0.4)  # Adjust height per row
-
-    fig, ax = plt.subplots(figsize=(fig_width, fig_height))
-    sns.heatmap(
-        pivot_df,
-        cmap=cmap,
-        linewidths=1,
-        linecolor="black",
-        norm=norm,
-        cbar=False,
-        square=True,
-        ax=ax,
-    )
-    plt.title(f"Heatmap for Model: {model_name}", fontsize=12)
-    plt.xlabel("Substring Index")
-    plt.ylabel("FSM (States, Alphabet)")
-    plt.xticks(rotation=45)
-
-    sns.despine(ax=ax, top=True, right=True, left=True, bottom=True)
-
-    return fig
-
-
-def generate_heatmap_for_intersection_model(model_name):
-    global intersection_df
-
-    cmap = ListedColormap(["lightblue", "red", "green"])
-    bounds = [-1.5, -0.5, 0.5, 1.5]
-    norm = BoundaryNorm(bounds, cmap.N)
-
-    # Filter for a specific model
-    model_df = intersection_df[intersection_df["model_name"] == model_name].copy()
-
-    if model_df.empty:
-        print(f"No data found for model {model_name}. Skipping heatmap generation.")
-        return None
-
-    model_df["fsm_info"] = model_df.apply(
-        lambda x: f"{x['num_states']} states, {x['num_alphabet']} alphabet", axis=1
-    )
-    model_df = model_df.sort_values(by=["num_states", "num_alphabet"])
-
-    pivot_df = (
-        model_df.pivot_table(
-            index="fsm_info",
-            columns="substring_index",
-            values="parsed_judge_response",
-            aggfunc="first",
-        )
-        .fillna(-1)
-        .astype(float)
-    )
-
-    # Dynamically adjust figure size
-    num_rows, num_cols = pivot_df.shape
-    fig_width = max(12, num_cols * 0.5)
-    fig_height = max(8, num_rows * 0.4)
-
-    fig, ax = plt.subplots(figsize=(fig_width, fig_height))
-    sns.heatmap(
-        pivot_df,
-        cmap=cmap,
-        linewidths=1,
-        linecolor="black",
-        norm=norm,
-        cbar=False,
-        square=True,
-        ax=ax,
-    )
-    plt.title(f"Heatmap for Model: {model_name}", fontsize=12)
-    plt.xlabel("Substring Index")
-    plt.ylabel("FSM (States, Alphabet)")
-    plt.xticks(rotation=45)
-
-    sns.despine(ax=ax, top=True, right=True, left=True, bottom=True)
-
-    plt.close(fig)
-    return fig
-
-
-def show_constraint_heatmap(evt: gr.SelectData):
-    model_name = evt.value
-    return generate_heatmap_for_specific_model(model_name)
-
-
-def show_constraint_heatmap_cot(evt: gr.SelectData):
-    model_name = evt.value
-    return generate_heatmap_for_specific_model_cot(model_name)
-
-
-def show_intersection_heatmap(evt: gr.SelectData):
-    model_name = evt.value
-    return generate_heatmap_for_intersection_model(model_name)
-
-
 with gr.Blocks() as demo:
     gr.Markdown("# FSM Benchmark Leaderboard")
     with gr.Tab("Text-only Benchmark"):
-        gr.Markdown("# Text-only Leaderboard (Judged by Qwen)")
-        leader_board = gr.Dataframe(accuracy_df_qwen, headers=headers_with_icons)
+        leader_board = gr.Dataframe(accuracy_df, headers=headers_with_icons)
         gr.Markdown("## Heatmap")
         heatmap_image_qwen = gr.Image(label="", show_label=False)
-        leader_board.select(fn=load_heatmap_qwen, outputs=[heatmap_image_qwen])
+        leader_board.select(fn=load_heatmap, outputs=[heatmap_image_qwen])
 
-    with gr.Tab("Vision Benchmark", visible=False):
-        gr.Markdown("# Vision Benchmark Leaderboard")
-        leader_board_vision = gr.Dataframe(
-            vision_accuracy_df, headers=headers_with_icons
-        )
-        gr.Markdown("## Heatmap")
-        heatmap_image_vision = gr.Image(label="", show_label=False)
-        leader_board_vision.select(
-            fn=load_vision_heatmap, outputs=[heatmap_image_vision]
-        )
-
-    with gr.Tab("Text-only Benchmark (CoT)", visible=False):
-        gr.Markdown("# Text-only Leaderboard (CoT)")
-        cot_leader_board_text = gr.Dataframe(
-            cot_text_accuracy_df, headers=headers_with_icons
-        )
-        gr.Markdown("## Heatmap")
-        cot_heatmap_image_text = gr.Image(label="", show_label=False)
-        cot_leader_board_text.select(
-            fn=load_cot_heatmap, outputs=[cot_heatmap_image_text]
-        )
-
-    # with gr.Tab("Vision Benchmark (CoT)"):
-    #     gr.Markdown("# Vision Benchmark Leaderboard (CoT)")
-    #     cot_leader_board_vision = gr.Dataframe(
-    #         cot_vision_accuracy_df, headers=headers_with_icons
+    # with gr.Tab("Vision Benchmark", visible=False):
+    #     gr.Markdown("# Vision Benchmark Leaderboard")
+    #     leader_board_vision = gr.Dataframe(
+    #         vision_accuracy_df, headers=headers_with_icons
     #     )
     #     gr.Markdown("## Heatmap")
-    #     cot_heatmap_image_vision = gr.Image(label="", show_label=False)
-    #     cot_leader_board_vision.select(
-    #         fn=load_cot_vision_heatmap, outputs=[cot_heatmap_image_vision]
+    #     heatmap_image_vision = gr.Image(label="", show_label=False)
+    #     leader_board_vision.select(
+    #         fn=load_vision_heatmap, outputs=[heatmap_image_vision]
     #     )
 
-    with gr.Tab("Constraint Text-only Results"):
-        gr.Markdown("## Constraint Text-only Leaderboard by first substring")
-        included_models = gr.CheckboxGroup(
-            label="Models to include",
-            choices=all_text_only_model_names,
-            value=all_text_only_model_names,
-            interactive=True,
-        )
-        with gr.Row():
-            number_of_queries = gr.Textbox(label="Number of included queries")
-
-            number_of_fsms = gr.Textbox(label="Number of included  FSMs")
-
-        constrained_leader_board_text = gr.Dataframe()
-        constrained_leader_board_plot = gr.Plot()
-
-    included_models.select(
-        fn=calculate_order_by_first_substring,
-        inputs=[included_models],
-        outputs=[constrained_leader_board_text, number_of_queries, number_of_fsms],
-        queue=True,
-    )
-
-    with gr.Tab("Constraint Text-only Results (CoT)", visible=False):
-        gr.Markdown("## Constraint Text-only Leaderboard by first substrin (CoT)")
-        included_models_cot = gr.CheckboxGroup(
-            label="Models to include",
-            choices=all_cot_text_only_models,
-            value=all_cot_text_only_models,
-            interactive=True,
-        )
-        with gr.Row():
-            number_of_queries_cot = gr.Textbox(label="Number of included queries")
-            number_of_fsms_cot = gr.Textbox(label="Number of included  FSMs")
-
-        constrained_leader_board_text_cot = gr.Dataframe()
-        constrained_leader_board_plot_cot = gr.Plot()
-
-    with gr.Tab("Majority Vote (Subset 1)", visible=False):
-        gr.Markdown("## Majority Vote (Subset 1)")
-        intersection_leader_board = gr.Dataframe(
-            intersection_df_acc, headers=headers_with_icons
-        )
-        heatmap_image = gr.Plot(label="Model Heatmap")
-
-    with gr.Tab("Text-only Benchmark (deprecated)", visible=False):
-        gr.Markdown("# Text-only Leaderboard")
-        leader_board = gr.Dataframe(accuracy_df, headers=headers_with_icons)
-        gr.Markdown("## Heatmap")
-        heatmap_image = gr.Image(label="", show_label=False)
-        leader_board.select(fn=load_heatmap, outputs=[heatmap_image])
-
-    # ============ Callbacks ============
-
-    included_models_cot.select(
-        fn=calculate_order_by_first_substring_cot,
-        inputs=[included_models_cot],
-        outputs=[
-            constrained_leader_board_text_cot,
-            number_of_queries_cot,
-            number_of_fsms_cot,
-        ],
-        queue=True,
-    )
-
-    constrained_leader_board_text.select(
-        fn=show_constraint_heatmap, outputs=[constrained_leader_board_plot]
-    )
-
-    constrained_leader_board_text_cot.select(
-        fn=show_constraint_heatmap_cot, outputs=[constrained_leader_board_plot_cot]
-    )
-
-    intersection_leader_board.select(
-        fn=show_intersection_heatmap, outputs=[heatmap_image]
-    )
+    # with gr.Tab("Text-only Benchmark (CoT)", visible=False):
+    #     gr.Markdown("# Text-only Leaderboard (CoT)")
+    #     cot_leader_board_text = gr.Dataframe(
+    #         cot_text_accuracy_df, headers=headers_with_icons
+    #     )
+    #     gr.Markdown("## Heatmap")
+    #     cot_heatmap_image_text = gr.Image(label="", show_label=False)
+    #     cot_leader_board_text.select(
+    #         fn=load_cot_heatmap, outputs=[cot_heatmap_image_text]
+    #     )
+
+    # with gr.Tab("Constraint Text-only Results (CoT)", visible=False):
+    #     gr.Markdown("## Constraint Text-only Leaderboard by first substrin (CoT)")
+    #     included_models_cot = gr.CheckboxGroup(
+    #         label="Models to include",
+    #         choices=all_cot_text_only_models,
+    #         value=all_cot_text_only_models,
+    #         interactive=True,
+    #     )
+    #     with gr.Row():
+    #         number_of_queries_cot = gr.Textbox(label="Number of included queries")
+    #         number_of_fsms_cot = gr.Textbox(label="Number of included  FSMs")
+
+    #     constrained_leader_board_text_cot = gr.Dataframe()
+    #     constrained_leader_board_plot_cot = gr.Plot()
+
+    # with gr.Tab("Majority Vote (Subset 1)", visible=False):
+    #     gr.Markdown("## Majority Vote (Subset 1)")
+    #     intersection_leader_board = gr.Dataframe(
+    #         intersection_df_acc, headers=headers_with_icons
+    #     )
+    #     heatmap_image = gr.Plot(label="Model Heatmap")
+
+    # with gr.Tab("Text-only Benchmark (deprecated)", visible=False):
+    #     gr.Markdown("# Text-only Leaderboard")
+    #     leader_board = gr.Dataframe(accuracy_df, headers=headers_with_icons)
+    #     gr.Markdown("## Heatmap")
+    #     heatmap_image = gr.Image(label="", show_label=False)
+    #     leader_board.select(fn=load_heatmap, outputs=[heatmap_image])
+
+    # # ============ Callbacks ============
+
+    # included_models_cot.select(
+    #     fn=calculate_order_by_first_substring_cot,
+    #     inputs=[included_models_cot],
+    #     outputs=[
+    #         constrained_leader_board_text_cot,
+    #         number_of_queries_cot,
+    #         number_of_fsms_cot,
+    #     ],
+    #     queue=True,
+    # )
+
+    # constrained_leader_board_text.select(
+    #     fn=show_constraint_heatmap, outputs=[constrained_leader_board_plot]
+    # )
+
+    # constrained_leader_board_text_cot.select(
+    #     fn=show_constraint_heatmap_cot, outputs=[constrained_leader_board_plot_cot]
+    # )
+
+    # intersection_leader_board.select(
+    #     fn=show_intersection_heatmap, outputs=[heatmap_image]
+    # )
 
     demo.launch()

results-cot/CodeLlama-70b-Instruct-hf.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a8d72952877e3e4023251396d037ebae8145e3e82e2d4a328ce132171ea70267
-size 20756425

results-cot/Mixtral-8x7B-Instruct-v0.1.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:093e919d90609c3be8d6818cf56ca018214da3a42b78aeaf85f92581b72c5ad4
-size 19494123

results-cot/Mixtral-8x7B-Instruct-v0.1.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:686692584c6ba027c454d699bbf585b95e5c99bfc426810ea74b327a975b9cf3
-size 19489822

results-cot/Qwen1.5-72B-Chat.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:32681449776facf1084405001e69ed7926b79c69f9717fb159e3eb064b333636
-size 15795431

results-cot/Qwen1.5-72B-Chat.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1c20383298d4b6482ca7c30bf91822e24099dc67b71a3be10271005e25208c40
-size 15778970

results-cot/claude-3-sonnet-20240229.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:77c47e8698b8aa86de42bda82cb27c1efc0174c18b3ba306ff263cbd260de20e
-size 13144187

results-cot/claude-3-sonnet-20240229.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7e5a528d8fa3491adbc4e2e5a982e65a8a9c2fb5a96d321c864935dbe506453e
-size 13047413

results-cot/dbrx-instruct.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:698626edcabf06c89b9b8fac2f929927e9b7351306f525cb87fc6e06ce1bc3e3
-size 19267224

results-cot/deepseek-llm-67b-chat.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:191a06465559524615f8cca0c46ca2af417a289e9fbab2109e2d2a3c92432fe2
-size 16692090

results-cot/deepseek-llm-67b-chat.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:005500788ecc004bb7a86054f8921e6331addd52c62d8e611a9d82b649ed4925
-size 16712146

results-cot/gemini-pro.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cd52b8cfe861dd9fc106dafbe11569f9f2bf5848482e9b42c0ad3e88ffc83035
-size 14773471

results-cot/gemini-pro.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:72caa2e3c4ef872b3d6065f55179aa85d3eaa7978f9abb6fc1e3f58db19d8a69
-size 14759970

results-cot/gemma-7b-it.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f8535fa3f2ef5a94b1b552859930e0476ca0f3c77ec4c277893a9ab9ef45d6c3
-size 16793758

results-cot/gemma-7b-it.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c581027f8b78df5934117276cec3e53613f5ac953d045f71af4121b3ec2e1a4
-size 16822239

results-cot/gpt-3.5-turbo-0125.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e9ea8fd5fd1e335397c14e3b312f60111614de4c20e50e5f3bc9d4fb6f049d70
-size 14471960

results-cot/gpt-3.5-turbo-0125.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:702c5dd6ffb22938a59815c65759ca5a201369b59633ee6c80c7030151e53634
-size 14487477

results-cot/gpt-4-turbo-2024-04-09.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:011eeea335d67be1a859d51ca2accfd6bc56a2222394db52e68c9f444fec1c9b
-size 20522713

results-cot/gpt-4-turbo-2024-04-09.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e1383c823ccc8e09830dd4b96742c19fdf7515b56a715edd6ce131ff07fa029c
-size 20513069

results-vision/claude-3-opus-20240229.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:74ed9f652dc23c7408b307cb477e91692b561739bf23b33ba93319d9d073f294
-size 10855333

results-vision/claude-3-opus-20240229.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:db495f6b9f264c3d9978c7d479b8206fbf8e67843c302c42416e0c6ee1aaa0a4
-size 10789555

results-vision/claude-3-opus-vision.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:db495f6b9f264c3d9978c7d479b8206fbf8e67843c302c42416e0c6ee1aaa0a4
-size 10789555

results-vision/gemini-pro-vision.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a33999636396dc5a3dfc858de2b0a421a6fa48d78d51ad74838f2d6afc261999
-size 6158469

results-vision/gemini-pro-vision.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6dec98de51c833c57d8a322c9084391b0aeeffd67a6ee0edbed2f23061aeb1e0
-size 6118424

results-vision/gpt-4v.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e98609b7b262836c49cdaf5d3dfc02b7037dc9fcc1f75a41d58a984015318759
-size 6363780

results/CodeLlama-70b-Instruct-hf.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3726905a1656174f3c29edfced6f2eec63222f6be8965c0d970264901d8cfc75
-size 16476347