plotly plot

README.md

@@ -13,12 +13,21 @@ pinned: false
 
 
 ```
-@misc{chalamalasetti2023clembench,
-      title={Clembench: Using Game Play to Evaluate Chat-Optimized Language Models as Conversational Agents}, 
-      author={Kranti Chalamalasetti and Jana Götze and Sherzod Hakimov and Brielen Madureira and Philipp Sadler and David Schlangen},
-      year={2023},
-      eprint={2305.13455},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
+@inproceedings{chalamalasetti-etal-2023-clembench,
+    title = "clembench: Using Game Play to Evaluate Chat-Optimized Language Models as Conversational Agents",
+    author = {Chalamalasetti, Kranti  and
+      G{\"o}tze, Jana  and
+      Hakimov, Sherzod  and
+      Madureira, Brielen  and
+      Sadler, Philipp  and
+      Schlangen, David},
+      booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
+      month = dec,
+      year = "2023",
+      address = "Singapore",
+      publisher = "Association for Computational Linguistics",
+      url = "https://aclanthology.org/2023.emnlp-main.689",
+      pages = "11174--11219"
 }
+
 ```

app.py

@@ -1,29 +1,31 @@
 import gradio as gr
 
 from src.assets.text_content import TITLE, INTRODUCTION_TEXT
-from src.utils import get_data, compare_plots, filter_search
+from src.leaderboard_utils import filter_search, get_github_data
+from src.plot_utils import split_models, compare_plots
 
-############################ For Leaderboards #############################
-DATA_PATH = 'versions'
-latest_flag = True #Set flag to iclude latest data in Details and Versions Tab
-latest_df, latest_vname, previous_df, previous_vname = get_data(DATA_PATH, latest_flag)
+# For Leaderboards
+# Get CSV data
+global primary_leaderboard_df, version_dfs, version_names
+primary_leaderboard_df, version_dfs, version_names = get_github_data()
 
 global prev_df
-prev_df = previous_df[0]
+prev_df = version_dfs[0]
 def select_prev_df(name):
-    ind = previous_vname.index(name)
-    prev_df = previous_df[ind]
+    ind = version_names.index(name)
+    prev_df = version_dfs[ind]
     return prev_df
 
-############################ For Plots ####################################
-global plot_df, MODEL_COLS
-plot_df = latest_df[0]
-MODEL_COLS = list(plot_df['Model'].unique())
+# For Plots 
+global plot_df, OPEN_MODELS, CLOSED_MODELS, SHOW_ALL, SHOW_NAMES
+plot_df = primary_leaderboard_df[0]
+MODELS = list(plot_df[list(plot_df.columns)[0]].unique())
+OPEN_MODELS, CLOSED_MODELS = split_models(MODELS)
 
 
-############# MAIN APPLICATION ######################
-demo = gr.Blocks()
-with demo:
+# MAIN APPLICATION s
+main_app = gr.Blocks()
+with main_app:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
@@ -35,17 +37,17 @@ with demo:
                     show_label=False,
                     elem_id="search-bar",
                 )
-                        
+                                    
             leaderboard_table = gr.components.Dataframe(
-                value=latest_df[0],
+                value=primary_leaderboard_df[0],
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=True,
             )
 
-            # Add a dummy leaderboard to handle search queries from the latest_df and not update latest_df
+            # Add a dummy leaderboard to handle search queries from the primary_leaderboard_df and not update primary_leaderboard_df
             dummy_leaderboard_table = gr.components.Dataframe(
-                value=latest_df[0],
+                value=primary_leaderboard_df[0],
                 elem_id="leaderboard-table",
                 interactive=False,
                 visible=False,
@@ -57,36 +59,88 @@ with demo:
                 leaderboard_table,
                 queue=True
             )
+
         with gr.TabItem("📈 Plot", id=3):
             with gr.Row():
-                model_cols = gr.CheckboxGroup(
-                    MODEL_COLS, 
-                    label="Select Models 🤖", 
+                open_models_selection = gr.CheckboxGroup(
+                    OPEN_MODELS, 
+                    label="Open-weight Models 🌐",
+                    value=[],
+                    elem_id="value-select",
+                    interactive=True,
+                )
+
+            with gr.Row():
+                closed_models_selection = gr.CheckboxGroup(
+                    CLOSED_MODELS, 
+                    label="Closed-weight Models 💼",
                     value=[],
-                    elem_id="column-select",
+                    elem_id="value-select-2",
                     interactive=True,
                 )
+            
+            with gr.Row():
+                with gr.Column():
+                    show_all = gr.CheckboxGroup(
+                        ["Select All Models"],
+                        label="Show plot for all models 🤖",
+                        value=[],
+                        elem_id="value-select-3",
+                        interactive=True,
+                    )
+                
+                with gr.Column():
+                    show_names = gr.CheckboxGroup(
+                        ["Show Names"],
+                        label ="Show names of models on the plot 🏷️",
+                        value=[],
+                        elem_id="value-select-4",
+                        interactive=True,
+                    ) 
 
             with gr.Row():
-                plot_grdf = gr.DataFrame(
+                dummy_plot_df = gr.DataFrame(
                     value=plot_df,
                     visible=False
                 )
+
             with gr.Row():
-                # Output block for the plot
-                plot_output = gr.Plot()
+                with gr.Column():
+                    # Output block for the plot
+                    plot_output = gr.Plot()
+
+            open_models_selection.change(
+                compare_plots,
+                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names],
+                plot_output,
+                queue=True
+            )
+
+            closed_models_selection.change(
+                compare_plots,
+                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names],
+                plot_output,
+                queue=True
+            )
+            
+            show_all.change(
+                compare_plots,
+                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names],
+                plot_output,
+                queue=True
+            )
 
-            model_cols.change(
+            show_names.change(
                 compare_plots,
-                [plot_grdf, model_cols],
+                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names],
                 plot_output,
                 queue=True
             )
 
         with gr.TabItem("🔄 Versions and Details", elem_id="details", id=2):
             with gr.Row():
-                ver_selection = gr.Dropdown(
-                    previous_vname, label="Select Version 🕹️", value=previous_vname[0]
+                version_select = gr.Dropdown(
+                    version_names, label="Select Version 🕹️", value=version_names[0]
                 )
             with gr.Row():
                 search_bar_prev = gr.Textbox(
@@ -116,13 +170,14 @@ with demo:
                 queue=True
             )
 
-            ver_selection.change(
+            version_select.change(
                 select_prev_df,
-                [ver_selection],
+                [version_select],
                 prev_table,
                 queue=True
             )
+    main_app.load()
+
+main_app.queue()
 
-    demo.load()
-demo.queue()
-demo.launch()
+main_app.launch()

requirements.txt

@@ -42,7 +42,7 @@ orjson==3.8.10
 packaging==23.1
 pandas==2.0.0
 Pillow==9.5.0
-plotly==5.14.1
+plotly==5.18.0
 pyarrow==11.0.0
 pydantic==1.10.7
 pydub==0.25.1

src/assets/text_content.py

@@ -1,13 +1,14 @@
 TITLE = """<h1 align="center" id="space-title"> 🏆 CLEM Leaderboard</h1>"""
 
 INTRODUCTION_TEXT = """
+<h6 align="center">
 The CLEM Leaderboard aims to track, rank and evaluate current cLLMs (chat-optimized Large Language Models) with the suggested pronounciation “clems”. 
 
 The benchmarking approach is described in [Clembench: Using Game Play to Evaluate Chat-Optimized Language Models as Conversational Agents](https://arxiv.org/abs/2305.13455).
 
-Source code benchmarking "clems" is available here: [Clembench](https://github.com/clembench/clembench)
+Source code for benchmarking "clems" is available here: [Clembench](https://github.com/clembench/clembench)
 
-All generated files and results from the benchmark runs are available here: [clembench-runs](https://github.com/clembench/clembench-runs)
+All generated files and results from the benchmark runs are available here: [clembench-runs](https://github.com/clembench/clembench-runs) </h6>
 """
 
 SHORT_NAMES = {

src/leaderboard_utils.py

@@ -0,0 +1,137 @@
+import os
+import pandas as pd
+import requests, json
+from io import StringIO
+
+def get_github_data():
+    '''
+    Get data from csv files on Github
+    Args:
+        None    
+    Returns: 
+        latest_df: singular list containing dataframe of the latest version of the leaderboard with only 4 columns 
+        all_dfs: list of dataframes for previous versions + latest version including columns for all games
+        all_vnames: list of the names for the previous versions + latest version (For Details and Versions Tab Dropdown)
+    '''
+    uname = "clembench"
+    repo = "clembench-runs"
+    json_url = f"https://raw.githubusercontent.com/{uname}/{repo}/main/benchmark_runs.json"
+    resp = requests.get(json_url)
+    if resp.status_code == 200:
+        json_data = json.loads(resp.text)
+        versions = json_data['versions']
+        version_names = []
+        csv_url = f"https://raw.githubusercontent.com/{uname}/{repo}/main/"
+        for ver in versions:
+            version_names.append(ver['version'])
+            csv_path = ver['result_file'].split('/')[1:]
+            csv_path = '/'.join(csv_path)
+        
+        #Sort by latest version
+        float_content = [float(s[1:]) for s in version_names]
+        float_content.sort(reverse=True)
+        version_names = ['v'+str(s) for s in float_content]
+
+        DFS = []
+        for version in version_names:
+            result_url = csv_url+ version + '/' + csv_path
+            csv_response = requests.get(result_url)
+            if csv_response.status_code == 200:
+                df = pd.read_csv(StringIO(csv_response.text))
+                df = process_df(df)
+                df = df.sort_values(by=list(df.columns)[1], ascending=False) # Sort by clemscore
+                DFS.append(df)
+            else:
+                print(f"Failed to read CSV file for version : {version}. Status Code : {resp.status_code}")
+
+        # Only keep relavant columns for the main leaderboard
+        latest_df_dummy = DFS[0]
+        all_columns = list(latest_df_dummy.columns)
+        keep_columns = all_columns[0:4]
+        latest_df_dummy = latest_df_dummy.drop(columns=[c for c in all_columns if c not in keep_columns])
+
+        latest_df = [latest_df_dummy]
+        all_dfs = []
+        all_vnames = []
+        for df, name in zip(DFS, version_names):
+            all_dfs.append(df)
+            all_vnames.append(name) 
+        return latest_df, all_dfs, all_vnames
+    
+    else:
+        print(f"Failed to read JSON file: Status Code : {resp.status_code}")
+
+def process_df(df: pd.DataFrame) -> pd.DataFrame:
+    '''
+    Process dataframe 
+    - Remove repition in model names 
+    - Convert datatypes to sort by "float" instead of "str" for sorting
+    - Update column names
+    Args:
+        df: Unprocessed Dataframe (after using update_cols)
+    Returns:
+        df: Processed Dataframe
+    '''
+
+    # Change column type to float from str
+    list_column_names = list(df.columns)
+    model_col_name = list_column_names[0]
+    for col in list_column_names:
+        if col != model_col_name:
+            df[col] = df[col].astype(float)
+
+    # Remove repetition in model names, if any
+    models_list = []
+    for i in range(len(df)):
+        model_name = df.iloc[i][model_col_name]
+        splits = model_name.split('--')
+        splits = [split.replace('-t0.0', '') for split in splits] # Comment to not remove -t0.0
+        if splits[0] == splits[1]:
+            models_list.append(splits[0])
+        else:
+            models_list.append(splits[0] + "--" + splits[1])
+    df[model_col_name] = models_list
+
+    # Update column names
+    update = ['Model', 'Clemscore', '% Played', 'Quality Score']
+    game_metrics = list_column_names[4:]
+
+    for col in game_metrics:
+        splits = col.split(',')
+        update.append(splits[0].capitalize() + "" + splits[1])
+    
+    map_cols = {}
+    for i in range(len(update)):
+        map_cols[list_column_names[i]] = str(update[i])
+
+    df = df.rename(columns=map_cols)    
+    return df
+
+def filter_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    '''
+    Filter the dataframe based on the search query
+    Args:
+        df: Unfiltered dataframe
+        query: a string of queries separated by ";"
+    Return:
+        filtered_df: Dataframe containing searched queries in the 'Model' column 
+    '''
+    queries = query.split(';')
+    list_cols = list(df.columns)
+    df_len = len(df)
+    filtered_models = []
+    models_list = list(df[list_cols[0]])
+    for q in queries:
+        q = q.lower()
+        q = q.strip()
+        for i in range(df_len):
+            model_name = models_list[i]
+            if q in model_name.lower():
+                filtered_models.append(model_name) # Append model names containing query q
+
+    filtered_df = df[df[list_cols[0]].isin(filtered_models)]
+
+    if query == "":
+        return df
+
+    return filtered_df

src/plot_utils.py

@@ -0,0 +1,129 @@
+import pandas as pd
+import plotly.express as px
+
+from src.assets.text_content import SHORT_NAMES
+
+def plotly_plot(df:pd.DataFrame, LIST:list, ALL:list, NAMES:list):
+    '''
+    Takes in a list of models for a plotly plot
+    Args:
+        df: A dummy dataframe of latest version
+        LIST: List of models to plot
+        ALL: Either [] or ["Show All Models"] - toggle view to plot all models 
+        NAMES: Either [] or ["Show Names"] - toggle view to show model names on plot 
+    Returns:
+        Fig: plotly figure
+    '''
+    
+    # Get list of all models and append short names column to df
+    list_columns = list(df.columns)
+    ALL_LIST = list(df[list_columns[0]].unique())
+    short_names = label_map(ALL_LIST)
+    list_short_names = list(short_names.values())
+    df["Short"] = list_short_names
+
+    if ALL:
+        LIST = ALL_LIST
+    # Filter dataframe based on the provided list of models
+    df = df[df[list_columns[0]].isin(LIST)]
+    
+
+    if NAMES:
+        fig = px.scatter(df, x=list_columns[2], y=list_columns[3], color=list_columns[0], symbol=list_columns[0],
+                 color_discrete_map={"category1": "blue", "category2": "red"},
+                 hover_name=list_columns[0], template="plotly_white", text="Short")
+        fig.update_traces(textposition='top center')
+    else:
+        fig = px.scatter(df, x=list_columns[2], y=list_columns[3], color=list_columns[0], symbol=list_columns[0],
+                    color_discrete_map={"category1": "blue", "category2": "red"},
+                    hover_name=list_columns[0], template="plotly_white")
+        
+    fig.update_layout(
+        xaxis_title='% Played',
+        yaxis_title='Quality Score',
+        title='Overview of benchmark results',
+        height=1000,
+        width=1500
+    )
+
+    fig.update_xaxes(range=[-5, 105])
+    fig.update_yaxes(range=[-5, 105])
+
+    return fig
+
+
+# ['Model', 'Clemscore', 'All(Played)', 'All(Quality Score)']
+def compare_plots(df: pd.DataFrame, LIST1: list, LIST2: list, ALL:list, NAMES:list):
+    '''
+    Quality Score v/s % Played plot by selecting models
+    Args:
+        df: A dummy dataframe of latest version
+        LIST1: The list of open source models to show in the plot, updated from frontend
+        LIST2: The list of commercial models to show in the plot, updated from frontend
+        ALL: Either [] or ["Show All Models"] - toggle view to plot all models 
+        NAMES: Either [] or ["Show Names"] - toggle view to show model names on plot 
+    Returns:
+        fig: The plot
+    '''
+
+    # Combine lists for Open source and commercial models
+    LIST = LIST1 + LIST2
+    fig = plotly_plot(df, LIST, ALL, NAMES)
+
+    return fig
+    
+def shorten_model_name(full_name):
+    # Split the name into parts
+    parts = full_name.split('-')
+
+    # Process the name parts to keep only the parts with digits (model sizes and versions)
+    short_name_parts = [part for part in parts if any(char.isdigit() for char in part)]
+
+    if len(parts) == 1:
+        short_name = ''.join(full_name[0:min(3, len(full_name))])
+    else:
+        # Join the parts to form the short name
+        short_name = '-'.join(short_name_parts)
+
+        # Remove any leading or trailing hyphens
+        short_name = full_name[0] + '-'+ short_name.strip('-')
+
+    return short_name
+
+def label_map(model_list: list) -> dict:
+    '''
+    Generate a map from long names to short names, to plot them in frontend graph
+    Define the short names in src/assets/text_content.py
+    Args: 
+        model_list: A list of long model names
+    Returns:
+        short_name: A dict from long to short name
+    '''
+    short_names = {}
+    for model_name in model_list:
+        if model_name in SHORT_NAMES:
+            short_name = SHORT_NAMES[model_name]
+        else:
+            short_name = shorten_model_name(model_name)
+
+        # Define the short name and indicate both models are same
+        short_names[model_name] = short_name
+
+    return short_names
+    
+def split_models(MODEL_LIST: list):
+    '''
+    Split the models into open source and commercial
+    '''
+    open_models = []
+    comm_models = []
+
+    for model in MODEL_LIST:
+        if model.startswith(('gpt-', 'claude-', 'command')):
+            comm_models.append(model)
+        else:
+            open_models.append(model)
+
+    open_models.sort(key=lambda o: o.upper())
+    comm_models.sort(key=lambda c: c.upper())
+    return open_models, comm_models

src/utils.py

@@ -1,238 +0,0 @@
-import os
-import pandas as pd
-import matplotlib.pyplot as plt
-import numpy as np
-
-from src.assets.text_content import SHORT_NAMES
-
-def update_cols(df: pd.DataFrame) -> pd.DataFrame:
-    '''
-    Change three header rows to a single header row
-    Args:
-        df: Raw dataframe containing 3 separate header rows
-            Remove this function if the dataframe has only one header row
-
-    Returns:
-        df: Updated dataframe which has only 1 header row instead of 3
-    '''
-    default_cols = list(df.columns)
-
-    # First 4 columns are initalised in 'update', Append additional columns for games Model, Clemscore, ALL(PLayed) and ALL(Main Score)
-    update = ['Model', 'Clemscore', 'Played', 'Quality Score']
-    game_metrics = default_cols[4:]
-
-    # Change columns Names for each Game
-    for i in range(len(game_metrics)):
-        if i%3 == 0:
-            game = game_metrics[i]
-            update.append(str(game).capitalize() + "(Played)")
-            update.append(str(game).capitalize() + "(Quality Score)") 
-            update.append(str(game).capitalize() + "(Quality Score[std])")
-
-    # Create a dict to change names of the columns
-    map_cols = {}
-    for i in range(len(default_cols)):
-        map_cols[default_cols[i]] = str(update[i])
-
-    df = df.rename(columns=map_cols)
-    df = df.iloc[2:]
-
-    return df
-
-def process_df(df: pd.DataFrame) -> pd.DataFrame:
-    '''
-    Process dataframe - Remove repition in model names, convert datatypes to sort by "float" instead of "str"
-    Args:
-        df: Unprocessed Dataframe (after using update_cols)
-    Returns:
-        df: Processed Dataframe
-    '''
-
-    # Change column type to float from str
-    list_column_names = list(df.columns)
-    model_col_name = list_column_names[0]
-    for col in list_column_names:
-        if col != model_col_name:
-            df[col] = df[col].astype(float)
-
-    # Remove repetition in model names, if any
-    models_list = []
-    for i in range(len(df)):
-        model_name = df.iloc[i][model_col_name]
-        splits = model_name.split('--')
-        splits = [split.replace('-t0.0', '') for split in splits] # Comment to not remove -t0.0
-        if splits[0] == splits[1]:
-            models_list.append(splits[0])
-        else:
-            models_list.append(splits[0] + "--" + splits[1])
-    df[model_col_name] = models_list
-    
-    return df
-
-def get_data(path: str, flag: bool):
-    '''
-    Get a list of all version names and respective Dataframes 
-    Args: 
-        path: Path to the directory containing CSVs of different versions -> v0.9.csv, v1.0.csv, ....
-        flag: Set this flag to include the latest version in Details and Versions tab
-    Returns: 
-        latest_df: singular list containing dataframe of the latest version of the leaderboard with only 4 columns 
-        latest_vname: list of the name of latest version 
-        previous_df: list of dataframes for previous versions (can skip latest version if required) 
-        previous_vname: list of the names for the previous versions (INCLUDED IN Details and Versions Tab)
-
-    '''
-    # Check if Directory is empty
-    list_versions = os.listdir(path)
-    if not list_versions:
-        print("Directory is empty")
-
-    else:
-        files = [file for file in list_versions if file.endswith('.csv')]
-        files.sort(reverse=True)
-        file_names = [os.path.splitext(file)[0] for file in files]
-
-        DFS = []
-        for file in files:
-            df = pd.read_csv(os.path.join(path, file))
-            df = update_cols(df) # Remove if by default there is only one header row
-            df = process_df(df) # Process Dataframe
-            df = df.sort_values(by=list(df.columns)[1], ascending=False) # Sort by clemscore
-            DFS.append(df)
-
-        # Only keep relavant columns for the main leaderboard
-        latest_df_dummy = DFS[0]
-        all_columns = list(latest_df_dummy.columns)
-        keep_columns = all_columns[0:4]
-        latest_df_dummy = latest_df_dummy.drop(columns=[c for c in all_columns if c not in keep_columns])
-
-        latest_df = [latest_df_dummy]
-        latest_vname = [file_names[0]]
-        previous_df = []
-        previous_vname = []
-        for df, name in zip(DFS, file_names):
-            previous_df.append(df)
-            previous_vname.append(name) 
-        
-        if not flag:
-            previous_df.pop(0)
-            previous_vname.pop(0)
-
-        return latest_df, latest_vname, previous_df, previous_vname
-    
-    return None
-
-
-# ['Model', 'Clemscore', 'All(Played)', 'All(Quality Score)']
-def compare_plots(df: pd.DataFrame, LIST: list):
-    '''
-    Quality Score v/s % Played plot by selecting models
-    Args:
-        LIST: The list of models to show in the plot, updated from frontend
-    Returns:
-        fig: The plot
-    '''
-    short_names = label_map(LIST)
-
-    list_columns = list(df.columns)
-    df = df[df[list_columns[0]].isin(LIST)]
-
-    X = df[list_columns[2]]
-    fig, ax = plt.subplots()
-    for model in LIST:
-        short = short_names[model]
-        # same_flag = short_names[model][1]
-        model_df = df[df[list_columns[0]] == model]
-        x = model_df[list_columns[2]]
-        y = model_df[list_columns[3]]
-        color = plt.cm.rainbow(x / max(X))  # Use a colormap for different colors
-        plt.scatter(x, y, color=color)
-        # if same_flag:
-        plt.annotate(f'{short}', (x, y), textcoords="offset points", xytext=(0, -15), ha='center', rotation=0)
-        # else:
-        #     plt.annotate(f'{short}', (x, y), textcoords="offset points", xytext=(20, -3), ha='center', rotation=0)
-    ax.grid(which='both', color='grey', linewidth=1, linestyle='-', alpha=0.2)
-    ax.set_xticks(np.arange(0,110,10))
-    plt.xlim(-10, 110)
-    plt.ylim(-10, 110)
-    plt.xlabel('% Played')
-    plt.ylabel('Quality Score')
-    plt.title('Overview of benchmark results')
-    plt.show()
-
-    return fig
-
-def shorten_model_name(full_name):
-    # Split the name into parts
-    parts = full_name.split('-')
-
-    # Process the name parts to keep only the parts with digits (model sizes and versions)
-    short_name_parts = [part for part in parts if any(char.isdigit() for char in part)]
-
-    if len(parts) == 1:
-        short_name = ''.join(full_name[0:min(3, len(full_name))])
-    else:
-        # Join the parts to form the short name
-        short_name = '-'.join(short_name_parts)
-
-        # Remove any leading or trailing hyphens
-        short_name = full_name[0] + '-'+ short_name.strip('-')
-
-    return short_name
-
-def label_map(model_list: list) -> dict:
-    '''
-    Generate a map from long names to short names, to plot them in frontend graph
-    Define the short names in src/assets/text_content.py
-    Args: 
-        model_list: A list of long model names
-    Returns:
-        short_name: A map from long to list of short name + indication if models are same or different
-    '''
-    short_names = {}
-    for model_name in model_list:
-        # splits = model_name.split('--')
-        # if len(splits) != 1:
-        #     splits[0] = SHORT_NAMES[splits[0] + '-']
-        #     splits[1] = SHORT_NAMES[splits[1] + '-']
-        #     # Define the short name and indicate there are two different models
-        #     short_names[model_name] = [splits[0] + '--' + splits[1], 0]
-        # else:
-        if model_name in SHORT_NAMES:
-            short_name = SHORT_NAMES[model_name]
-        else:
-            short_name = shorten_model_name(model_name)
-
-        # Define the short name and indicate both models are same
-        short_names[model_name] = short_name
-
-    return short_names
-
-def filter_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    '''
-    Filter the dataframe based on the search query
-    Args:
-        df: Unfiltered dataframe
-        query: a string of queries separated by ";"
-    Return:
-        filtered_df: Dataframe containing searched queries in the 'Model' column 
-    '''
-    queries = query.split(';')
-    list_cols = list(df.columns)
-    df_len = len(df)
-    filtered_models = []
-    models_list = list(df[list_cols[0]])
-    for q in queries:
-        q = q.lower()
-        for i in range(df_len):
-            model_name = models_list[i]
-            if q in model_name.lower():
-                filtered_models.append(model_name) # Append model names containing query q
-
-    filtered_df = df[df[list_cols[0]].isin(filtered_models)]
-
-    if query == "":
-        return df
-
-    return filtered_df
-    

versions/v0.9.csv

@@ -1,14 +0,0 @@
-,-,all,all,imagegame,imagegame,imagegame,privateshared,privateshared,privateshared,referencegame,referencegame,referencegame,taboo,taboo,taboo,wordle,wordle,wordle,wordle_withclue,wordle_withclue,wordle_withclue,wordle_withcritic,wordle_withcritic,wordle_withcritic
-,clemscore,Average % Played,Average Quality Score,% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std)
-model,,,,,,,,,,,,,,,,,,,,,,,,
-claude-v1.3-t0.0--claude-v1.3-t0.0,37.07,74.76,49.58,0.0,,,100.0,84.87,18.87,100.0,82.5,38.48,76.92,68.75,38.71,100.0,0.0,0.0,100.0,30.56,40.13,46.43,30.77,48.04
-falcon-40b-t0.0--falcon-40b-t0.0,0.71,0.95,75.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,3.33,50.0,,3.33,100.0,
-gpt-3.5-turbo-t0.0--gpt-3.5-turbo-t0.0,37.02,85.86,43.12,97.5,60.28,25.95,64.0,72.83,13.07,100.0,55.0,50.38,69.49,71.95,44.79,100.0,0.0,0.0,93.33,28.57,46.0,76.67,13.19,30.16
-gpt-3.5-turbo-t0.0--gpt-4-t0.0,42.39,86.75,48.87,97.5,64.95,25.45,,,,100.0,57.5,50.06,69.49,62.6,45.15,,,,,,,80.0,10.42,17.42
-gpt-4-t0.0--gpt-3.5-turbo-t0.0,55.62,82.78,67.19,65.0,81.0,21.54,,,,100.0,47.5,50.57,66.1,93.59,23.45,,,,,,,100.0,46.67,42.92
-gpt-4-t0.0--gpt-4-t0.0,59.49,96.06,61.93,77.5,89.06,22.28,100.0,90.79,8.2,100.0,75.0,43.85,94.92,76.19,37.45,100.0,3.67,8.4,100.0,49.67,42.09,100.0,49.11,38.46
-koala-13b-t0.0--koala-13b-t0.0,1.48,14.76,10.0,0.0,,,0.0,,,0.0,,,0.0,,,86.67,0.0,0.0,16.67,20.0,44.72,0.0,,
-luminous-supreme-t0.0--luminous-supreme-t0.0,0.0,16.24,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,3.33,0.0,,10.34,0.0,0.0
-oasst-12b-t0.0--oasst-12b-t0.0,1.74,20.85,8.33,0.0,,,0.0,,,15.0,33.33,51.64,0.0,,,100.0,0.0,0.0,16.67,0.0,0.0,14.29,0.0,0.0
-text-davinci-003-t0.0--text-davinci-003-t0.0,15.78,44.5,35.46,57.5,38.7,27.78,16.0,14.1,25.21,82.5,36.36,48.85,28.81,76.47,43.72,66.67,1.25,5.59,36.67,31.36,38.99,23.33,50.0,50.0
-vicuna-13b-t0.0--vicuna-13b-t0.0,4.24,13.58,31.25,0.0,,,0.0,,,0.0,,,5.08,100.0,0.0,56.67,0.0,0.0,13.33,25.0,50.0,20.0,0.0,0.0

versions/v1.0.csv

@@ -1,34 +0,0 @@
-,-,all,all,imagegame,imagegame,imagegame,privateshared,privateshared,privateshared,referencegame,referencegame,referencegame,taboo,taboo,taboo,wordle,wordle,wordle,wordle_withclue,wordle_withclue,wordle_withclue,wordle_withcritic,wordle_withcritic,wordle_withcritic
-,clemscore,Average % Played,Average Quality Score,% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std),% Played,Quality Score,Quality Score (std)
-model,,,,,,,,,,,,,,,,,,,,,,,,
-CodeLlama-34b-Instruct-hf-t0.0--CodeLlama-34b-Instruct-hf-t0.0,10.34,23.96,43.15,7.5,37.67,36.83,0.0,,,15.0,83.33,40.82,74.58,29.55,44.87,53.33,0.0,0.0,13.33,8.33,16.66,4.0,100.0,
-Mistral-7B-Instruct-v0.1-t0.0--Mistral-7B-Instruct-v0.1-t0.0,2.72,17.5,15.56,,,,20.0,0.0,0.0,,,,50.85,46.67,49.01,0.0,,,16.67,0.0,0.0,0.0,,
-Wizard-Vicuna-13B-Uncensored-HF-t0.0--Wizard-Vicuna-13B-Uncensored-HF-t0.0,2.06,9.49,21.71,0.0,,,0.0,,,7.5,66.67,57.74,22.03,30.77,48.04,3.33,0.0,,30.0,11.11,33.33,3.57,0.0,
-WizardLM-13b-v1.2-t0.0--WizardLM-13b-v1.2-t0.0,7.82,40.49,19.31,0.0,,,26.0,21.37,20.34,100.0,35.0,48.3,47.46,51.79,48.08,33.33,0.0,0.0,43.33,7.69,27.74,33.33,0.0,0.0
-WizardLM-70b-v1.0-t0.0--WizardLM-70b-v1.0-t0.0,16.7,51.65,32.34,0.0,,,24.0,62.3,19.84,100.0,32.5,47.43,62.71,67.57,47.46,60.0,0.0,0.0,70.0,21.43,40.53,44.83,10.26,28.49
-Yi-34B-Chat-t0.0--Yi-34B-Chat-t0.0,16.77,63.76,26.3,100.0,21.25,17.87,0.0,,,100.0,27.5,45.22,79.66,46.81,50.44,86.67,0.0,0.0,50.0,28.89,45.19,30.0,33.33,50.0
-claude-2-t0.0--claude-2-t0.0,33.71,82.12,41.05,0.0,,,100.0,75.89,18.57,100.0,45.0,50.38,91.53,73.15,39.71,100.0,0.0,0.0,90.0,21.6,36.56,93.33,30.65,43.22
-claude-2.1-t0.0--claude-2.1-t0.0,36.38,83.08,43.79,0.0,,,100.0,73.01,23.8,100.0,55.0,50.38,94.92,69.35,41.41,100.0,0.67,3.65,93.33,29.29,40.95,93.33,35.42,41.74
-claude-instant-1.2-t0.0--claude-instant-1.2-t0.0,15.44,59.61,25.91,0.0,,,96.0,34.37,30.79,0.0,,,77.97,60.51,44.77,100.0,0.0,0.0,90.0,18.52,37.08,53.33,16.15,28.94
-claude-v1.3-t0.0--claude-v1.3-t0.0,37.64,74.24,50.7,0.0,,,100.0,84.99,18.63,100.0,85.0,36.16,79.66,72.34,37.31,96.67,0.0,0.0,100.0,31.11,39.81,43.33,30.77,48.04
-command-t0.0--command-t0.0,3.12,10.01,31.13,0.0,,,0.0,,,2.5,0.0,,47.46,17.86,39.0,0.0,,,16.67,6.67,14.91,3.45,100.0,
-falcon-7b-instruct-t0.0--falcon-7b-instruct-t0.0,0.0,14.29,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,0.0,,,0.0,,
-gpt-3.5-turbo-0613-t0.0--gpt-3.5-turbo-0613-t0.0,32.53,91.96,35.37,97.5,62.49,30.09,91.84,21.17,25.65,100.0,55.0,50.38,64.41,63.16,47.48,100.0,0.0,0.0,100.0,21.38,38.8,90.0,24.38,35.8
-gpt-3.5-turbo-1106-t0.0--gpt-3.5-turbo-1106-t0.0,30.45,77.12,39.49,40.0,51.25,34.69,46.94,27.41,33.38,100.0,52.5,50.57,72.88,76.74,39.86,100.0,0.0,0.0,86.67,30.9,44.36,93.33,37.62,45.9
-gpt-4-0314-t0.0--gpt-4-0314-t0.0,58.81,93.79,62.7,65.0,88.92,16.24,100.0,91.12,7.48,100.0,77.5,42.29,91.53,79.63,32.48,100.0,2.78,7.37,100.0,50.78,41.69,100.0,48.17,41.42
-gpt-4-0613-t0.0--gpt-4-0613-t0.0,60.9,97.22,62.64,97.5,97.28,10.38,100.0,97.34,5.02,100.0,80.0,40.51,83.05,81.97,29.03,100.0,4.25,8.62,100.0,46.11,41.85,100.0,31.5,38.1
-gpt-4-1106-preview-t0.0--gpt-4-1106-preview-t0.0,60.33,97.95,61.59,97.5,94.15,14.85,100.0,83.25,12.51,100.0,90.0,30.38,88.14,83.97,29.88,100.0,7.5,13.01,100.0,49.11,42.93,100.0,23.17,37.74
-gpt4all-13b-snoozy-t0.0--gpt4all-13b-snoozy-t0.0,0.0,2.92,0.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,13.33,0.0,0.0,7.14,0.0,0.0
-koala-13B-HF-t0.0--koala-13B-HF-t0.0,1.25,23.22,5.38,0.0,,,0.0,,,0.0,,,52.54,16.13,37.39,100.0,0.0,0.0,10.0,0.0,0.0,0.0,,
-llama-2-13b-chat-hf-t0.0--llama-2-13b-chat-hf-t0.0,1.89,3.43,55.09,0.0,,,24.0,55.09,33.68,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,
-llama-2-70b-chat-hf-t0.0--llama-2-70b-chat-hf-t0.0,1.39,3.79,36.74,0.0,,,14.0,13.48,11.98,12.5,60.0,54.77,0.0,,,0.0,,,0.0,,,0.0,,
-llama-2-7b-chat-hf-t0.0--llama-2-7b-chat-hf-t0.0,0.24,6.05,4.0,0.0,,,0.0,,,0.0,,,42.37,4.0,20.0,0.0,,,0.0,,,0.0,,
-oasst-sft-4-pythia-12b-epoch-3.5-t0.0--oasst-sft-4-pythia-12b-epoch-3.5-t0.0,0.0,14.76,0.0,0.0,,,0.0,,,0.0,,,0.0,,,100.0,0.0,0.0,3.33,0.0,,0.0,,
-openchat_3.5-t0.0--openchat_3.5-t0.0,19.72,57.57,34.26,27.5,7.73,13.87,78.0,69.9,29.92,100.0,37.5,49.03,50.85,55.0,49.74,76.67,0.0,0.0,53.33,29.69,43.99,16.67,40.0,54.77
-sheep-duck-llama-2-13b-t0.0--sheep-duck-llama-2-13b-t0.0,6.74,34.86,19.34,0.0,,,0.0,,,97.5,33.33,47.76,89.83,0.0,0.0,0.0,,,23.33,19.05,37.8,33.33,25.0,42.49
-sheep-duck-llama-2-70b-v1.1-t0.0--sheep-duck-llama-2-70b-v1.1-t0.0,17.12,40.82,41.93,40.0,23.19,28.06,0.0,,,35.0,57.14,51.36,59.32,74.29,44.34,34.78,0.0,0.0,63.33,43.86,43.82,53.33,53.12,49.9
-vicuna-13b-v1.5-t0.0--vicuna-13b-v1.5-t0.0,7.21,34.74,20.74,0.0,,,24.0,0.0,0.0,80.0,28.12,45.68,49.15,37.93,49.38,26.67,0.0,0.0,36.67,30.3,45.84,26.67,28.12,45.19
-vicuna-33b-v1.3-t0.0--vicuna-33b-v1.3-t0.0,9.15,17.47,52.36,15.0,23.67,24.34,40.0,34.58,26.47,0.0,,,37.29,50.0,51.18,0.0,,,23.33,53.57,46.61,6.67,100.0,0.0
-vicuna-7b-v1.5-t0.0--vicuna-7b-v1.5-t0.0,3.46,12.86,26.91,0.0,,,4.0,0.0,0.0,0.0,,,62.71,24.32,43.5,0.0,,,13.33,50.0,57.74,10.0,33.33,57.74
-zephyr-7b-alpha-t0.0--zephyr-7b-alpha-t0.0,0.75,7.51,10.0,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,33.33,20.0,42.16,19.23,0.0,0.0
-zephyr-7b-beta-t0.0--zephyr-7b-beta-t0.0,1.23,3.95,31.25,0.0,,,0.0,,,0.0,,,0.0,,,0.0,,,13.33,25.0,50.0,14.29,37.5,47.87