update

.gitattributes

@@ -45,3 +45,39 @@ results/Claude-3-Haiku.csv filter=lfs diff=lfs merge=lfs -text
 results/Mixtral-8x7B-Instruct-0.1.csv filter=lfs diff=lfs merge=lfs -text
 results/CodeLlama-70B.csv filter=lfs diff=lfs merge=lfs -text
 results/Gemma-7B.csv filter=lfs diff=lfs merge=lfs -text
+results/Yi-34B-Chat.csv filter=lfs diff=lfs merge=lfs -text
+results/Claude-3-Haiku.jpg filter=lfs diff=lfs merge=lfs -text
+results/Gemma-7B.pkl filter=lfs diff=lfs merge=lfs -text
+results/Llama-2-70b-chat-hf.pkl filter=lfs diff=lfs merge=lfs -text
+results/Mistral-7B-Instruct-v0.2.png filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-0.1.png filter=lfs diff=lfs merge=lfs -text
+results/Claude-3-Opus.jpg filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70B.pkl filter=lfs diff=lfs merge=lfs -text
+results/Gemma-7B.jpg filter=lfs diff=lfs merge=lfs -text
+results/Mistral-7B-Instruct-v0.2.pkl filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-0.1.jpg filter=lfs diff=lfs merge=lfs -text
+results/Claude-3-Haiku.png filter=lfs diff=lfs merge=lfs -text
+results/GPT-3.5-Turbo.jpg filter=lfs diff=lfs merge=lfs -text
+results/GPT-3.5-Turbo.pkl filter=lfs diff=lfs merge=lfs -text
+results/Qwen1.5-72B-Chat.jpg filter=lfs diff=lfs merge=lfs -text
+results/StripedHyena-Nous-7B.pkl filter=lfs diff=lfs merge=lfs -text
+results/Yi-34B-Chat.png filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70B.png filter=lfs diff=lfs merge=lfs -text
+results/GPT-3.5-Turbo.png filter=lfs diff=lfs merge=lfs -text
+results/Llama-2-70b-chat-hf.jpg filter=lfs diff=lfs merge=lfs -text
+results/StripedHyena-Nous-7B.png filter=lfs diff=lfs merge=lfs -text
+results/Llama-2-70b-chat-hf.png filter=lfs diff=lfs merge=lfs -text
+results/Mistral-7B-Instruct-v0.2.jpg filter=lfs diff=lfs merge=lfs -text
+results/StripedHyena-Nous-7B.jpg filter=lfs diff=lfs merge=lfs -text
+results/Qwen1.5-72B-Chat.png filter=lfs diff=lfs merge=lfs -text
+results/Yi-34B-Chat.pkl filter=lfs diff=lfs merge=lfs -text
+results/CodeLlama-70B.jpg filter=lfs diff=lfs merge=lfs -text
+results/GPT-4-0125-preview.jpg filter=lfs diff=lfs merge=lfs -text
+results/Mixtral-8x7B-Instruct-0.1.pkl filter=lfs diff=lfs merge=lfs -text
+results/Yi-34B-Chat.jpg filter=lfs diff=lfs merge=lfs -text
+results/Claude-3-Haiku.pkl filter=lfs diff=lfs merge=lfs -text
+results/Gemma-7B.png filter=lfs diff=lfs merge=lfs -text
+results/Qwen1.5-72B-Chat.pkl filter=lfs diff=lfs merge=lfs -text
+results/Claude-3-Opus.pkl filter=lfs diff=lfs merge=lfs -text
+results/Claude-3-Opus.png filter=lfs diff=lfs merge=lfs -text
+results/GPT-4-0125-preview.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -3,9 +3,9 @@ import pandas as pd
 from glob import glob
 
 
-csv_results = glob("results/*.csv")
+csv_results = glob("results/*.pkl")
 # load the csv files into a dict with keys being name of the file and values being the data
-data = {file: pd.read_csv(file) for file in csv_results}
+data = {file: pd.read_pickle(file) for file in csv_results}
 
 
 def calculate_accuracy(df):
@@ -41,7 +41,7 @@ for file, df in data.items():
     # Get the breakdown accuracy and round each value
     breakdown_accuracy = [round(acc, 2) for acc in accuracy_breakdown(df)]
     # Prepare the model name from the file name
-    model_name = file.split("/")[-1].replace(".csv", "")  # Corrected the file extension
+    model_name = file.split("/")[-1].replace(".pkl", "")  # Corrected the file extension
     # Append the data to the list
     data_for_df.append([model_name, overall_accuracy] + breakdown_accuracy)
 
@@ -61,10 +61,25 @@ accuracy_df.columns = headers_with_icons
 accuracy_df.sort_values(by="⭐ Overall", ascending=False, inplace=True)
 
 
+def load_heatmap(evt: gr.SelectData):
+    print(
+        f"You selected {evt.value} at row {evt.index[0]}, column {evt.index[1]} from the Dataframe."
+    )
+
+    heatmap_image = gr.Image(f"results/{evt.value}.jpg")
+    return heatmap_image
+
+
 with gr.Blocks() as demo:
-    gr.Markdown("# FSMBench Leaderboard")
+    gr.Markdown("# FSM Benchmark Leaderboard")
     # add link to home page and dataset
 
     leader_board = gr.Dataframe(accuracy_df, headers=headers_with_icons)
 
+    gr.Markdown("## Heatmap")
+
+    heatamp_image = gr.Image(label="", show_label=False)
+
+    leader_board.select(fn=load_heatmap, outputs=[heatamp_image])
+
     demo.launch()

results/{Claude-3-Haiku.csv → Claude-3-Haiku.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45623535997485afdee5b0312f2b5fdcc26cf531fbb56b6c3af6e126dfbe7b0f
-size 19570166
+oid sha256:06b6c5cf244b982fcc45365fbeff804642130b80ec6f6f9b0cafcf69cb13e9fc
+size 19540844

results/{CodeLlama-70B.csv → Claude-3-Opus.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3726905a1656174f3c29edfced6f2eec63222f6be8965c0d970264901d8cfc75
-size 16476347
+oid sha256:424425308bdce6ebf2b59825b1b8b2796421792c44dafae8f74df06a07f3ef19
+size 20115450

results/CodeLlama-70B.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c0496d8536b01f37858e37e1b34eedd25efa1f205035868883bfcdc2ae6fb88
+size 16436822

results/GPT-3.5-Turbo.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8038b3c4d84ba654e8d2aebb41bd803efc0d477051a81f6fd2be95302e4a0c1d
+size 9470933

results/GPT-4-0125-preview.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2d4cbbacdff8172888d8d5e8917680f524d7cd73dcbcc7aa8d0e54c0246a752c
-size 18088521

results/Gemma-7B.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3bdc088d6c7eb18257ac35c1d2b2ee9f9849a69950016f6e9a0bf04be48a5ae2
-size 12624700

results/Gemma-7B.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f02f45ce5d312717f22676deec9d4a321cce8b74321059620056f99064ee7a15
+size 12654079

results/Llama-2-70b-chat-hf.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:42a31de917b05ed5405474a348d072426474a8fb2ce7ff462dbb121e25f4b6ad
-size 20760268

results/Llama-2-70b-chat-hf.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7bfc053d1fcc58f5f22e03c813d02cd634235c44b9a351fe084d4a1f659186a
+size 20685075

results/Mistral-7B-Instruct-v0.2.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:29ad4985661fc41e659a631fc74ba433cd08a571048f11436ccf87ff74f0db09
-size 27242025

results/Mistral-7B-Instruct-v0.2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99fa146558b73614994e83a667371795a7de461aca6e3580ebb72761fa1758bb
+size 27226799

results/Mixtral-8x7B-Instruct-0.1.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a93e2b963a5ac8129b5284f3fd7987964ef96fa0e64194de704a3549c611de1f
-size 17978176

results/Mixtral-8x7B-Instruct-0.1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01909843a40e65ff7e340cedbad451ab2337c7423161a46342e83d384ec24162
+size 17979541

results/Qwen1.5-72B-Chat.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0ba395c0b55330f689827527831e57e50ae9d824b6635b2bb569713afcf26d4b
-size 14219193

results/Qwen1.5-72B-Chat.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec5494612aaef1b9ac05c580fdac67469fc7bb3129e66b80ab720afeb9c71f22
+size 14196803

results/StripedHyena-Nous-7B.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f662367ea0d33a368aaa7a72cfeed41d2f3dc05be6289a6fe485a028c7cb98d5
-size 29219512

results/StripedHyena-Nous-7B.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48481a400fcecb1ddc80f4ebbf4c235f6b645ab28ad57a72ed1688e3cf17c192
+size 29177951

results/Yi-34B-Chat.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff7da8d16b7c70f06c2035d65039610b722b3b12cf64cb2f0efa7fcd41e1a82a
+size 20489399