Spaces:
Running
Running
BestWishYsh
commited on
Commit
•
1d873c6
1
Parent(s):
bbecd5e
coda
Browse files- .gitignore +0 -13
- .pre-commit-config.yaml +0 -53
- Makefile +0 -13
- README.md +2 -0
- app.py +306 -313
- constants.py +49 -0
- file/ChronoMagic-Bench-Input.json +9 -0
- file/results_ChronoMagic-Bench-150.csv +16 -0
- file/results_ChronoMagic-Bench.csv +12 -0
- pyproject.toml +0 -13
- requirements.txt +2 -18
- src/about.py +0 -72
- src/display/css_html_js.py +0 -105
- src/display/formatting.py +0 -27
- src/display/utils.py +0 -135
- src/envs.py +0 -25
- src/leaderboard/read_evals.py +0 -196
- src/populate.py +0 -58
- src/submission/check_validity.py +0 -99
- src/submission/submit.py +0 -119
.gitignore
DELETED
@@ -1,13 +0,0 @@
|
|
1 |
-
auto_evals/
|
2 |
-
venv/
|
3 |
-
__pycache__/
|
4 |
-
.env
|
5 |
-
.ipynb_checkpoints
|
6 |
-
*ipynb
|
7 |
-
.vscode/
|
8 |
-
|
9 |
-
eval-queue/
|
10 |
-
eval-results/
|
11 |
-
eval-queue-bk/
|
12 |
-
eval-results-bk/
|
13 |
-
logs/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.pre-commit-config.yaml
DELETED
@@ -1,53 +0,0 @@
|
|
1 |
-
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
2 |
-
#
|
3 |
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4 |
-
# you may not use this file except in compliance with the License.
|
5 |
-
# You may obtain a copy of the License at
|
6 |
-
#
|
7 |
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8 |
-
#
|
9 |
-
# Unless required by applicable law or agreed to in writing, software
|
10 |
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11 |
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12 |
-
# See the License for the specific language governing permissions and
|
13 |
-
# limitations under the License.
|
14 |
-
|
15 |
-
default_language_version:
|
16 |
-
python: python3
|
17 |
-
|
18 |
-
ci:
|
19 |
-
autofix_prs: true
|
20 |
-
autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
|
21 |
-
autoupdate_schedule: quarterly
|
22 |
-
|
23 |
-
repos:
|
24 |
-
- repo: https://github.com/pre-commit/pre-commit-hooks
|
25 |
-
rev: v4.3.0
|
26 |
-
hooks:
|
27 |
-
- id: check-yaml
|
28 |
-
- id: check-case-conflict
|
29 |
-
- id: detect-private-key
|
30 |
-
- id: check-added-large-files
|
31 |
-
args: ['--maxkb=1000']
|
32 |
-
- id: requirements-txt-fixer
|
33 |
-
- id: end-of-file-fixer
|
34 |
-
- id: trailing-whitespace
|
35 |
-
|
36 |
-
- repo: https://github.com/PyCQA/isort
|
37 |
-
rev: 5.12.0
|
38 |
-
hooks:
|
39 |
-
- id: isort
|
40 |
-
name: Format imports
|
41 |
-
|
42 |
-
- repo: https://github.com/psf/black
|
43 |
-
rev: 22.12.0
|
44 |
-
hooks:
|
45 |
-
- id: black
|
46 |
-
name: Format code
|
47 |
-
additional_dependencies: ['click==8.0.2']
|
48 |
-
|
49 |
-
- repo: https://github.com/charliermarsh/ruff-pre-commit
|
50 |
-
# Ruff version.
|
51 |
-
rev: 'v0.0.267'
|
52 |
-
hooks:
|
53 |
-
- id: ruff
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Makefile
DELETED
@@ -1,13 +0,0 @@
|
|
1 |
-
.PHONY: style format
|
2 |
-
|
3 |
-
|
4 |
-
style:
|
5 |
-
python -m black --line-length 119 .
|
6 |
-
python -m isort .
|
7 |
-
ruff check --fix .
|
8 |
-
|
9 |
-
|
10 |
-
quality:
|
11 |
-
python -m black --check --line-length 119 .
|
12 |
-
python -m isort --check-only .
|
13 |
-
ruff check .
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
CHANGED
@@ -7,6 +7,8 @@ sdk: gradio
|
|
7 |
app_file: app.py
|
8 |
pinned: true
|
9 |
license: apache-2.0
|
|
|
|
|
10 |
---
|
11 |
|
12 |
# Start the configuration
|
|
|
7 |
app_file: app.py
|
8 |
pinned: true
|
9 |
license: apache-2.0
|
10 |
+
sdk_version: 4.36.1
|
11 |
+
short_description: 'A Benchmark for Metamorphic Evaluation of T2V Generation'
|
12 |
---
|
13 |
|
14 |
# Start the configuration
|
app.py
CHANGED
@@ -1,345 +1,338 @@
|
|
1 |
-
|
|
|
2 |
import gradio as gr
|
3 |
import pandas as pd
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
from
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
72 |
):
|
73 |
-
|
74 |
-
|
75 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
76 |
return df
|
77 |
|
|
|
|
|
|
|
|
|
|
|
|
|
78 |
|
79 |
-
def
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
|
84 |
-
always_here_cols = [
|
85 |
-
AutoEvalColumn.model_type_symbol.name,
|
86 |
-
AutoEvalColumn.model.name,
|
87 |
-
]
|
88 |
-
# We use COLS to maintain sorting
|
89 |
-
filtered_df = df[
|
90 |
-
always_here_cols + [c for c in COLS if c in df.columns and c in columns]
|
91 |
-
]
|
92 |
-
return filtered_df
|
93 |
-
|
94 |
-
|
95 |
-
def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
|
96 |
-
final_df = []
|
97 |
-
if query != "":
|
98 |
-
queries = [q.strip() for q in query.split(";")]
|
99 |
-
for _q in queries:
|
100 |
-
_q = _q.strip()
|
101 |
-
if _q != "":
|
102 |
-
temp_filtered_df = search_table(filtered_df, _q)
|
103 |
-
if len(temp_filtered_df) > 0:
|
104 |
-
final_df.append(temp_filtered_df)
|
105 |
-
if len(final_df) > 0:
|
106 |
-
filtered_df = pd.concat(final_df)
|
107 |
-
filtered_df = filtered_df.drop_duplicates(
|
108 |
-
subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
|
109 |
-
)
|
110 |
-
|
111 |
-
return filtered_df
|
112 |
-
|
113 |
-
|
114 |
-
def filter_models(
|
115 |
-
df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
|
116 |
-
) -> pd.DataFrame:
|
117 |
-
# Show all models
|
118 |
-
if show_deleted:
|
119 |
-
filtered_df = df
|
120 |
-
else: # Show only still on the hub models
|
121 |
-
filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
|
122 |
-
|
123 |
-
type_emoji = [t[0] for t in type_query]
|
124 |
-
filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
|
125 |
-
filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
|
126 |
-
|
127 |
-
numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
|
128 |
-
params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
|
129 |
-
mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
|
130 |
-
filtered_df = filtered_df.loc[mask]
|
131 |
|
132 |
-
|
|
|
|
|
|
|
133 |
|
|
|
134 |
|
135 |
-
demo = gr.Blocks(css=custom_css)
|
136 |
-
with demo:
|
137 |
-
gr.HTML(TITLE)
|
138 |
-
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
|
139 |
|
|
|
|
|
|
|
|
|
140 |
with gr.Tabs(elem_classes="tab-buttons") as tabs:
|
141 |
-
|
|
|
142 |
with gr.Row():
|
143 |
-
with gr.
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
)
|
150 |
-
with gr.Row():
|
151 |
-
shown_columns = gr.CheckboxGroup(
|
152 |
-
choices=[
|
153 |
-
c.name
|
154 |
-
for c in fields(AutoEvalColumn)
|
155 |
-
if not c.hidden and not c.never_hidden
|
156 |
-
],
|
157 |
-
value=[
|
158 |
-
c.name
|
159 |
-
for c in fields(AutoEvalColumn)
|
160 |
-
if c.displayed_by_default and not c.hidden and not c.never_hidden
|
161 |
-
],
|
162 |
-
label="Select columns to show",
|
163 |
-
elem_id="column-select",
|
164 |
-
interactive=True,
|
165 |
-
)
|
166 |
-
with gr.Row():
|
167 |
-
deleted_models_visibility = gr.Checkbox(
|
168 |
-
value=False, label="Show gated/private/deleted models", interactive=True
|
169 |
-
)
|
170 |
-
with gr.Column(min_width=320):
|
171 |
-
#with gr.Box(elem_id="box-filter"):
|
172 |
-
filter_columns_type = gr.CheckboxGroup(
|
173 |
-
label="Model types",
|
174 |
-
choices=[t.to_str() for t in ModelType],
|
175 |
-
value=[t.to_str() for t in ModelType],
|
176 |
-
interactive=True,
|
177 |
-
elem_id="filter-columns-type",
|
178 |
-
)
|
179 |
-
filter_columns_precision = gr.CheckboxGroup(
|
180 |
-
label="Precision",
|
181 |
-
choices=[i.value.name for i in Precision],
|
182 |
-
value=[i.value.name for i in Precision],
|
183 |
-
interactive=True,
|
184 |
-
elem_id="filter-columns-precision",
|
185 |
-
)
|
186 |
-
filter_columns_size = gr.CheckboxGroup(
|
187 |
-
label="Model sizes (in billions of parameters)",
|
188 |
-
choices=list(NUMERIC_INTERVALS.keys()),
|
189 |
-
value=list(NUMERIC_INTERVALS.keys()),
|
190 |
-
interactive=True,
|
191 |
-
elem_id="filter-columns-size",
|
192 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
193 |
|
194 |
-
|
195 |
-
value=
|
196 |
-
|
197 |
-
|
198 |
-
|
199 |
-
headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
|
200 |
-
datatype=TYPES,
|
201 |
-
elem_id="leaderboard-table",
|
202 |
interactive=False,
|
203 |
visible=True,
|
204 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
205 |
|
206 |
-
|
207 |
-
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
212 |
)
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
-
|
219 |
-
filter_columns_precision,
|
220 |
-
filter_columns_size,
|
221 |
-
deleted_models_visibility,
|
222 |
-
search_bar,
|
223 |
-
],
|
224 |
-
leaderboard_table,
|
225 |
)
|
226 |
-
|
227 |
-
|
228 |
-
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
filter_columns_size,
|
235 |
-
deleted_models_visibility,
|
236 |
-
search_bar,
|
237 |
-
],
|
238 |
-
leaderboard_table,
|
239 |
-
queue=True,
|
240 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
241 |
|
242 |
-
|
243 |
-
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
|
244 |
|
245 |
-
|
246 |
-
|
247 |
-
|
248 |
-
|
249 |
|
250 |
-
with gr.Column():
|
251 |
-
with gr.Accordion(
|
252 |
-
f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
|
253 |
-
open=False,
|
254 |
-
):
|
255 |
-
with gr.Row():
|
256 |
-
finished_eval_table = gr.components.Dataframe(
|
257 |
-
value=finished_eval_queue_df,
|
258 |
-
headers=EVAL_COLS,
|
259 |
-
datatype=EVAL_TYPES,
|
260 |
-
row_count=5,
|
261 |
-
)
|
262 |
-
with gr.Accordion(
|
263 |
-
f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
|
264 |
-
open=False,
|
265 |
-
):
|
266 |
-
with gr.Row():
|
267 |
-
running_eval_table = gr.components.Dataframe(
|
268 |
-
value=running_eval_queue_df,
|
269 |
-
headers=EVAL_COLS,
|
270 |
-
datatype=EVAL_TYPES,
|
271 |
-
row_count=5,
|
272 |
-
)
|
273 |
-
|
274 |
-
with gr.Accordion(
|
275 |
-
f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
|
276 |
-
open=False,
|
277 |
-
):
|
278 |
-
with gr.Row():
|
279 |
-
pending_eval_table = gr.components.Dataframe(
|
280 |
-
value=pending_eval_queue_df,
|
281 |
-
headers=EVAL_COLS,
|
282 |
-
datatype=EVAL_TYPES,
|
283 |
-
row_count=5,
|
284 |
-
)
|
285 |
with gr.Row():
|
286 |
-
gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
|
287 |
|
288 |
with gr.Row():
|
289 |
with gr.Column():
|
290 |
-
model_name_textbox = gr.Textbox(
|
291 |
-
|
292 |
-
|
293 |
-
|
294 |
-
label="Model
|
295 |
-
multiselect=False,
|
296 |
-
value=None,
|
297 |
-
interactive=True,
|
298 |
)
|
299 |
-
|
300 |
-
|
301 |
-
|
302 |
-
|
303 |
-
label="Precision",
|
304 |
-
multiselect=False,
|
305 |
-
value="float16",
|
306 |
-
interactive=True,
|
307 |
)
|
308 |
-
|
309 |
-
|
310 |
-
label="Weights type",
|
311 |
-
multiselect=False,
|
312 |
-
value="Original",
|
313 |
-
interactive=True,
|
314 |
)
|
315 |
-
base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
|
316 |
-
|
317 |
-
submit_button = gr.Button("Submit Eval")
|
318 |
-
submission_result = gr.Markdown()
|
319 |
-
submit_button.click(
|
320 |
-
add_new_eval,
|
321 |
-
[
|
322 |
-
model_name_textbox,
|
323 |
-
base_model_name_textbox,
|
324 |
-
revision_name_textbox,
|
325 |
-
precision,
|
326 |
-
weight_type,
|
327 |
-
model_type,
|
328 |
-
],
|
329 |
-
submission_result,
|
330 |
-
)
|
331 |
|
332 |
-
|
333 |
-
|
334 |
-
|
335 |
-
|
336 |
-
|
337 |
-
|
338 |
-
|
339 |
-
|
340 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
341 |
|
342 |
-
|
343 |
-
|
344 |
-
|
345 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
|
2 |
+
|
3 |
import gradio as gr
|
4 |
import pandas as pd
|
5 |
+
import json
|
6 |
+
import io
|
7 |
+
|
8 |
+
from constants import *
|
9 |
+
|
10 |
+
global data_component, data_component_150, filter_component
|
11 |
+
|
12 |
+
def upload_file(files):
|
13 |
+
file_paths = [file.name for file in files]
|
14 |
+
return file_paths
|
15 |
+
|
16 |
+
def compute_scores(input_data):
|
17 |
+
return [None, [
|
18 |
+
input_data["Average_MTScore"],
|
19 |
+
input_data["Average_CHScore"],
|
20 |
+
input_data["Average_GPT4o-MTScore"],
|
21 |
+
input_data["Average_UMT-FVD"],
|
22 |
+
input_data["Average_UMTScore"]
|
23 |
+
]]
|
24 |
+
|
25 |
+
def add_new_eval(
|
26 |
+
input_file,
|
27 |
+
model_name_textbox: str,
|
28 |
+
revision_name_textbox: str,
|
29 |
+
backbone_type_dropdown: str,
|
30 |
+
model_link: str,
|
31 |
+
):
|
32 |
+
if input_file is None:
|
33 |
+
return "Error! Empty file!"
|
34 |
+
else:
|
35 |
+
input_json = json.load(io.BytesIO(input_file))
|
36 |
+
|
37 |
+
if model_name_textbox not in input_json:
|
38 |
+
return f"Error! Model '{model_name_textbox}' not found in input file!"
|
39 |
+
|
40 |
+
selected_model_data = input_json[model_name_textbox]
|
41 |
+
|
42 |
+
scores = compute_scores(selected_model_data)
|
43 |
+
input_data = scores[1]
|
44 |
+
input_data = [float(i) for i in input_data]
|
45 |
+
|
46 |
+
csv_data = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH)
|
47 |
+
|
48 |
+
if revision_name_textbox == '':
|
49 |
+
col = csv_data.shape[0]
|
50 |
+
model_name = model_name_textbox
|
51 |
+
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']]
|
52 |
+
assert model_name not in name_list
|
53 |
+
else:
|
54 |
+
model_name = revision_name_textbox
|
55 |
+
model_name_list = csv_data['Model']
|
56 |
+
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list]
|
57 |
+
if revision_name_textbox not in name_list:
|
58 |
+
col = csv_data.shape[0]
|
59 |
+
else:
|
60 |
+
col = name_list.index(revision_name_textbox)
|
61 |
+
|
62 |
+
if model_link == '':
|
63 |
+
model_name = model_name # no url
|
64 |
+
else:
|
65 |
+
model_name = '[' + model_name + '](' + model_link + ')'
|
66 |
+
|
67 |
+
backbone = backbone_type_dropdown
|
68 |
+
|
69 |
+
new_data = [
|
70 |
+
model_name,
|
71 |
+
backbone,
|
72 |
+
input_data[3],
|
73 |
+
input_data[4],
|
74 |
+
input_data[0],
|
75 |
+
input_data[1],
|
76 |
+
input_data[2],
|
77 |
+
]
|
78 |
+
csv_data.loc[col] = new_data
|
79 |
+
csv_data.to_csv(CSV_DIR_CHRONOMAGIC_BENCH, index=False)
|
80 |
+
return "Evaluation successfully submitted!"
|
81 |
+
|
82 |
+
def add_new_eval_150(
|
83 |
+
input_file,
|
84 |
+
model_name_textbox: str,
|
85 |
+
revision_name_textbox: str,
|
86 |
+
backbone_type_dropdown: str,
|
87 |
+
model_link: str,
|
88 |
):
|
89 |
+
if input_file is None:
|
90 |
+
return "Error! Empty file!"
|
91 |
+
else:
|
92 |
+
input_json = json.load(io.BytesIO(input_file))
|
93 |
+
|
94 |
+
if model_name_textbox not in input_json:
|
95 |
+
return f"Error! Model '{model_name_textbox}' not found in input file!"
|
96 |
+
|
97 |
+
selected_model_data = input_json[model_name_textbox]
|
98 |
+
|
99 |
+
scores = compute_scores(selected_model_data)
|
100 |
+
input_data = scores[1]
|
101 |
+
input_data = [float(i) for i in input_data]
|
102 |
+
|
103 |
+
csv_data = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150)
|
104 |
+
|
105 |
+
if revision_name_textbox == '':
|
106 |
+
col = csv_data.shape[0]
|
107 |
+
model_name = model_name_textbox
|
108 |
+
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']]
|
109 |
+
assert model_name not in name_list
|
110 |
+
else:
|
111 |
+
model_name = revision_name_textbox
|
112 |
+
model_name_list = csv_data['Model']
|
113 |
+
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list]
|
114 |
+
if revision_name_textbox not in name_list:
|
115 |
+
col = csv_data.shape[0]
|
116 |
+
else:
|
117 |
+
col = name_list.index(revision_name_textbox)
|
118 |
+
|
119 |
+
if model_link == '':
|
120 |
+
model_name = model_name # no url
|
121 |
+
else:
|
122 |
+
model_name = '[' + model_name + '](' + model_link + ')'
|
123 |
+
|
124 |
+
backbone = backbone_type_dropdown
|
125 |
+
|
126 |
+
new_data = [
|
127 |
+
model_name,
|
128 |
+
backbone,
|
129 |
+
input_data[3],
|
130 |
+
input_data[4],
|
131 |
+
input_data[0],
|
132 |
+
input_data[1],
|
133 |
+
input_data[2],
|
134 |
+
]
|
135 |
+
csv_data.loc[col] = new_data
|
136 |
+
csv_data.to_csv(CSV_DIR_CHRONOMAGIC_BENCH_150, index=False)
|
137 |
+
return "Evaluation (150) successfully submitted!"
|
138 |
+
|
139 |
+
def get_baseline_df():
|
140 |
+
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH)
|
141 |
+
df = df.sort_values(by="MTScore↑", ascending=False)
|
142 |
+
present_columns = MODEL_INFO + checkbox_group.value
|
143 |
+
df = df[present_columns]
|
144 |
return df
|
145 |
|
146 |
+
def get_baseline_df_150():
|
147 |
+
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150)
|
148 |
+
df = df.sort_values(by="MTScore↑", ascending=False)
|
149 |
+
present_columns = MODEL_INFO + checkbox_group_150.value
|
150 |
+
df = df[present_columns]
|
151 |
+
return df
|
152 |
|
153 |
+
def get_all_df():
|
154 |
+
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH)
|
155 |
+
df = df.sort_values(by="MTScore↑", ascending=False)
|
156 |
+
return df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
157 |
|
158 |
+
def get_all_df_150():
|
159 |
+
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150)
|
160 |
+
df = df.sort_values(by="MTScore↑", ascending=False)
|
161 |
+
return df
|
162 |
|
163 |
+
block = gr.Blocks()
|
164 |
|
|
|
|
|
|
|
|
|
165 |
|
166 |
+
with block:
|
167 |
+
gr.Markdown(
|
168 |
+
LEADERBORAD_INTRODUCTION
|
169 |
+
)
|
170 |
with gr.Tabs(elem_classes="tab-buttons") as tabs:
|
171 |
+
# table 1
|
172 |
+
with gr.TabItem("🏅 ChronoMagic-Bench", elem_id="ChronoMagic-Bench-tab-table", id=0):
|
173 |
with gr.Row():
|
174 |
+
with gr.Accordion("Citation", open=False):
|
175 |
+
citation_button = gr.Textbox(
|
176 |
+
value=CITATION_BUTTON_TEXT,
|
177 |
+
label=CITATION_BUTTON_LABEL,
|
178 |
+
elem_id="citation-button",
|
179 |
+
show_copy_button=True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
180 |
)
|
181 |
+
|
182 |
+
gr.Markdown(
|
183 |
+
TABLE_INTRODUCTION
|
184 |
+
)
|
185 |
+
|
186 |
+
checkbox_group = gr.CheckboxGroup(
|
187 |
+
choices=ALL_RESULTS,
|
188 |
+
value=SELECTED_RESULTS,
|
189 |
+
label="Select options",
|
190 |
+
interactive=True,
|
191 |
+
)
|
192 |
|
193 |
+
data_component = gr.components.Dataframe(
|
194 |
+
value=get_baseline_df,
|
195 |
+
headers=COLUMN_NAMES,
|
196 |
+
type="pandas",
|
197 |
+
datatype=DATA_TITILE_TYPE,
|
|
|
|
|
|
|
198 |
interactive=False,
|
199 |
visible=True,
|
200 |
+
)
|
201 |
+
|
202 |
+
def on_checkbox_group_change(selected_columns):
|
203 |
+
selected_columns = [item for item in ALL_RESULTS if item in selected_columns]
|
204 |
+
present_columns = MODEL_INFO + selected_columns
|
205 |
+
updated_data = get_all_df()[present_columns]
|
206 |
+
updated_data = updated_data.sort_values(by=present_columns[1], ascending=False)
|
207 |
+
updated_headers = present_columns
|
208 |
+
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
|
209 |
+
|
210 |
+
filter_component = gr.components.Dataframe(
|
211 |
+
value=updated_data,
|
212 |
+
headers=updated_headers,
|
213 |
+
type="pandas",
|
214 |
+
datatype=update_datatype,
|
215 |
+
interactive=False,
|
216 |
+
visible=True,
|
217 |
+
)
|
218 |
+
|
219 |
+
return filter_component
|
220 |
+
|
221 |
+
checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
|
222 |
|
223 |
+
# table 2
|
224 |
+
with gr.TabItem("🏅 ChronoMagic-Bench-150", elem_id="ChronoMagic-Bench-150-tab-table", id=1):
|
225 |
+
with gr.Row():
|
226 |
+
with gr.Accordion("Citation", open=False):
|
227 |
+
citation_button = gr.Textbox(
|
228 |
+
value=CITATION_BUTTON_TEXT,
|
229 |
+
label=CITATION_BUTTON_LABEL,
|
230 |
+
elem_id="citation-button",
|
231 |
+
show_copy_button=True
|
232 |
+
)
|
233 |
+
|
234 |
+
gr.Markdown(
|
235 |
+
TABLE_INTRODUCTION
|
236 |
)
|
237 |
+
|
238 |
+
checkbox_group_150 = gr.CheckboxGroup(
|
239 |
+
choices=ALL_RESULTS,
|
240 |
+
value=SELECTED_RESULTS_150,
|
241 |
+
label="Select options",
|
242 |
+
interactive=True,
|
|
|
|
|
|
|
|
|
|
|
|
|
243 |
)
|
244 |
+
|
245 |
+
data_component_150 = gr.components.Dataframe(
|
246 |
+
value=get_baseline_df_150,
|
247 |
+
headers=COLUMN_NAMES,
|
248 |
+
type="pandas",
|
249 |
+
datatype=DATA_TITILE_TYPE,
|
250 |
+
interactive=False,
|
251 |
+
visible=True,
|
|
|
|
|
|
|
|
|
|
|
|
|
252 |
)
|
253 |
+
|
254 |
+
def on_checkbox_group_150_change(selected_columns):
|
255 |
+
selected_columns = [item for item in ALL_RESULTS if item in selected_columns]
|
256 |
+
present_columns = MODEL_INFO + selected_columns
|
257 |
+
updated_data = get_all_df_150()[present_columns]
|
258 |
+
updated_data = updated_data.sort_values(by=present_columns[1], ascending=False)
|
259 |
+
updated_headers = present_columns
|
260 |
+
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
|
261 |
+
|
262 |
+
filter_component = gr.components.Dataframe(
|
263 |
+
value=updated_data,
|
264 |
+
headers=updated_headers,
|
265 |
+
type="pandas",
|
266 |
+
datatype=update_datatype,
|
267 |
+
interactive=False,
|
268 |
+
visible=True,
|
269 |
+
)
|
270 |
+
|
271 |
+
return filter_component
|
272 |
|
273 |
+
checkbox_group_150.change(fn=on_checkbox_group_150_change, inputs=checkbox_group_150, outputs=data_component_150)
|
|
|
274 |
|
275 |
+
# table 3
|
276 |
+
with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=2):
|
277 |
+
with gr.Row():
|
278 |
+
gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
|
279 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
280 |
with gr.Row():
|
281 |
+
gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
|
282 |
|
283 |
with gr.Row():
|
284 |
with gr.Column():
|
285 |
+
model_name_textbox = gr.Textbox(
|
286 |
+
label="Model name", placeholder="MagicTime"
|
287 |
+
)
|
288 |
+
revision_name_textbox = gr.Textbox(
|
289 |
+
label="Revision Model Name", placeholder="MagicTime"
|
|
|
|
|
|
|
290 |
)
|
291 |
+
backbone_type_dropdown = gr.Dropdown(
|
292 |
+
label="Backbone Type",
|
293 |
+
choices=["DiT", "U-Net"],
|
294 |
+
value="DiT"
|
|
|
|
|
|
|
|
|
295 |
)
|
296 |
+
model_link = gr.Textbox(
|
297 |
+
label="Model Link", placeholder="https://github.com/PKU-YuanGroup/MagicTime"
|
|
|
|
|
|
|
|
|
298 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
299 |
|
300 |
+
with gr.Column():
|
301 |
+
input_file = gr.File(label="Click to Upload a json File", type='binary')
|
302 |
+
submit_button = gr.Button("Submit Eval (ChronoMagic-Bench)")
|
303 |
+
submit_button_150 = gr.Button("Submit Eval (ChronoMagic-Bench-150)")
|
304 |
+
|
305 |
+
submission_result = gr.Markdown()
|
306 |
+
submit_button.click(
|
307 |
+
add_new_eval,
|
308 |
+
inputs=[
|
309 |
+
input_file,
|
310 |
+
model_name_textbox,
|
311 |
+
revision_name_textbox,
|
312 |
+
backbone_type_dropdown,
|
313 |
+
model_link,
|
314 |
+
],
|
315 |
+
outputs=submission_result,
|
316 |
+
)
|
317 |
+
submit_button_150.click(
|
318 |
+
add_new_eval_150,
|
319 |
+
inputs=[
|
320 |
+
input_file,
|
321 |
+
model_name_textbox,
|
322 |
+
revision_name_textbox,
|
323 |
+
backbone_type_dropdown,
|
324 |
+
model_link,
|
325 |
+
],
|
326 |
+
outputs = submission_result,
|
327 |
+
)
|
328 |
|
329 |
+
with gr.Row():
|
330 |
+
data_run = gr.Button("Refresh")
|
331 |
+
data_run.click(
|
332 |
+
get_baseline_df, outputs=data_component
|
333 |
+
)
|
334 |
+
data_run.click(
|
335 |
+
get_baseline_df_150, outputs=data_component_150
|
336 |
+
)
|
337 |
+
|
338 |
+
block.launch()
|
constants.py
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
MODEL_INFO = ["Model", "Backbone"]
|
2 |
+
|
3 |
+
ALL_RESULTS = ["UMT-FVD↓", "UMTScore↑", "MTScore↑", "CHScore↑", "GPT4o-MTScore↑"]
|
4 |
+
|
5 |
+
SELECTED_RESULTS = ["UMT-FVD↓", "UMTScore↑", "MTScore↑", "CHScore↑", "GPT4o-MTScore↑"]
|
6 |
+
SELECTED_RESULTS_150 = ["UMT-FVD↓", "UMTScore↑", "MTScore↑", "GPT4o-MTScore↑"]
|
7 |
+
|
8 |
+
DATA_TITILE_TYPE = ["markdown", 'markdown', "number", "number", "number", "number", "number"]
|
9 |
+
|
10 |
+
CSV_DIR_CHRONOMAGIC_BENCH = "./file/results_ChronoMagic-Bench.csv"
|
11 |
+
CSV_DIR_CHRONOMAGIC_BENCH_150 = "./file/results_ChronoMagic-Bench-150.csv"
|
12 |
+
|
13 |
+
COLUMN_NAMES = MODEL_INFO + ALL_RESULTS
|
14 |
+
|
15 |
+
LEADERBORAD_INTRODUCTION = """# ChronoMagic-Bench Leaderboard
|
16 |
+
|
17 |
+
Welcome to the leaderboard of the ChronoMagic-Bench!
|
18 |
+
|
19 |
+
🏆ChronoMagic-Bench represents the inaugural benchmark dedicated to assessing T2V models' capabilities in generating time-lapse videos that demonstrate significant metamorphic amplitude and temporal coherence. The benchmark probes T2V models for their physics, biology, and chemistry capabilities, in a free-form text control.
|
20 |
+
|
21 |
+
Please refer to [our paper](https://arxiv.org/abs/2311.16103) for more details.
|
22 |
+
"""
|
23 |
+
|
24 |
+
SUBMIT_INTRODUCTION = """# Submit Introduction
|
25 |
+
Obtain `ChronoMagic-Bench-Input.json` from our [github repository](https://github.com/PKU-YuanGroup/Video-Bench#%EF%B8%8F-evaluate-your-own-model) after evaluation.
|
26 |
+
|
27 |
+
|
28 |
+
## Submit Example
|
29 |
+
For example, if you want to upload Video-ChatGPT's result in the leaderboard, you need to:
|
30 |
+
1. Fill in 'MagicTime' in 'Model Name' if it is your first time to submit your result (You can leave 'Revision Model Name' blank).
|
31 |
+
2. Fill in 'MagicTime' in 'Revision Model Name' if you want to update your result (You can leave 'Model Name' blank).
|
32 |
+
3. Select ‘Backbone Type’ (DiT or U-Net).
|
33 |
+
4. Fill in 'https://github.com/x/x' in 'Model Link'.
|
34 |
+
5. Upload `ChronoMagic-Bench-Input.json`.
|
35 |
+
6. Click the 'Submit Eval' button.
|
36 |
+
7. Click 'Refresh' to obtain the uploaded leaderboard.
|
37 |
+
"""
|
38 |
+
|
39 |
+
TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
|
40 |
+
We use UMT-FVD, UMTScore, MTScore, CHScore, GPT4o-MTScore as the primary evaluation metric for each tasks.
|
41 |
+
"""
|
42 |
+
|
43 |
+
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
|
44 |
+
CITATION_BUTTON_TEXT = r"""@article{yuan2024magictime,
|
45 |
+
title={MagicTime: Time-lapse Video Generation Models as Metamorphic Simulators},
|
46 |
+
author={Yuan, Shenghai and Huang, Jinfa and Shi, Yujun and Xu, Yongqi and Zhu, Ruijie and Lin, Bin and Cheng, Xinhua and Yuan, Li and Luo, Jiebo},
|
47 |
+
journal={arXiv preprint arXiv:2404.05014},
|
48 |
+
year={2024}
|
49 |
+
}"""
|
file/ChronoMagic-Bench-Input.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"test": {
|
3 |
+
"Average_MTScore": 0.2613712220142285,
|
4 |
+
"Average_CHScore": 4.1676077942528345,
|
5 |
+
"Average_GPT4o-MTScore": 2.3333333333333335,
|
6 |
+
"Average_UMT-FVD": -1,
|
7 |
+
"Average_UMTScore": -1
|
8 |
+
}
|
9 |
+
}
|
file/results_ChronoMagic-Bench-150.csv
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Model,Backbone,UMT-FVD↓,UMTScore↑,MTScore↑,CHScore↑,GPT4o-MTScore↑
|
2 |
+
[Gen-2 (20240610)](https://research.runwayml.com/gen2),DiT,218.99,2.4,0.373,5.27,2.62
|
3 |
+
[Pika-2.0 (20240610)](https://www.pika.art/),DiT,223.05,2.317,0.347,4.0,2.48
|
4 |
+
[Dream Machine (20240610)](https://lumalabs.ai/dream-machine),DiT,214.91,2.387,0.474,2.3,3.11
|
5 |
+
[KeLing (20240610)](https://h5.kwaiying.com/officialWebsite),DiT,202.32,2.517,0.369,3.69,2.74
|
6 |
+
[ModelScopeT2V](https://huggingface.co/ali-vilab/text-to-video-ms-1.7b),U-Net,230.74,2.783,0.409,10.64,3.01
|
7 |
+
[ZeroScope](https://huggingface.co/cerspense/zeroscope_v2_576w),U-Net,260.61,2.232,0.403,24.1,2.29
|
8 |
+
[T2V-Zero](https://github.com/Picsart-AI-Research/Text2Video-Zero),U-Net,250.22,2.559,0.399,1.84,2.62
|
9 |
+
[LaVie](https://github.com/Vchitect/LaVie),U-Net,210.39,2.714,0.35,9.58,2.5
|
10 |
+
[AnimateDiff-V3](https://github.com/guoyww/AnimateDiff),U-Net,239.31,2.837,0.47,11.09,2.62
|
11 |
+
[VideoCrafter2](https://github.com/AILab-CVC/VideoCrafter),U-Net,214.06,2.763,0.437,7.78,2.87
|
12 |
+
[MagicTime](https://github.com/PKU-YuanGroup/MagicTime),U-Net,294.72,1.763,0.479,11.58,3.05
|
13 |
+
[Latte](https://github.com/Vchitect/Latte),DiT,232.29,2.122,0.366,13.79,2.42
|
14 |
+
[OpenSora 1.1](https://github.com/hpcaitech/Open-Sora),DiT,241.09,2.676,0.448,10.46,2.57
|
15 |
+
[OpenSora 1.2](https://github.com/hpcaitech/Open-Sora),DiT,210.93,2.681,0.383,5.6,2.5
|
16 |
+
[OpenSoraPlan v1.1](https://github.com/PKU-YuanGroup/Open-Sora-Plan),DiT,228.7,2.459,0.331,10.32,2.21
|
file/results_ChronoMagic-Bench.csv
ADDED
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Model,Backbone,UMT-FVD↓,UMTScore↑,MTScore↑,CHScore↑,GPT4o-MTScore↑
|
2 |
+
[ModelScopeT2V](https://huggingface.co/ali-vilab/text-to-video-ms-1.7b),U-Net,194.77,2.909,0.401,11.03,2.86
|
3 |
+
[ZeroScope](https://huggingface.co/cerspense/zeroscope_v2_576w),U-Net,227.02,2.35,0.4,25.13,2.09
|
4 |
+
[T2V-Zero](https://github.com/Picsart-AI-Research/Text2Video-Zero),U-Net,209.66,2.661,0.4,1.68,2.55
|
5 |
+
[LaVie](https://github.com/Vchitect/LaVie),U-Net,166.97,2.763,0.346,8.6,2.46
|
6 |
+
[AnimateDiff-V3](https://github.com/guoyww/AnimateDiff),U-Net,197.89,2.944,0.467,11.36,2.62
|
7 |
+
[VideoCrafter2](https://github.com/AILab-CVC/VideoCrafter),U-Net,178.45,2.753,0.433,8.27,2.68
|
8 |
+
[MagicTime](https://github.com/PKU-YuanGroup/MagicTime),U-Net,257.56,1.916,0.478,10.66,3.13
|
9 |
+
[Latte](https://github.com/Vchitect/Latte),DiT,192.12,2.111,0.363,13.81,2.2
|
10 |
+
[OpenSora 1.1](https://github.com/hpcaitech/Open-Sora),DiT,195.43,2.678,0.444,10.03,2.52
|
11 |
+
[OpenSora 1.2](https://github.com/hpcaitech/Open-Sora),DiT,166.92,2.781,0.375,4.69,2.56
|
12 |
+
[OpenSoraPlan v1.1](https://github.com/PKU-YuanGroup/Open-Sora-Plan),DiT,188.53,2.421,0.327,10.35,2.19
|
pyproject.toml
DELETED
@@ -1,13 +0,0 @@
|
|
1 |
-
[tool.ruff]
|
2 |
-
# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
|
3 |
-
select = ["E", "F"]
|
4 |
-
ignore = ["E501"] # line too long (black is taking care of this)
|
5 |
-
line-length = 119
|
6 |
-
fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
|
7 |
-
|
8 |
-
[tool.isort]
|
9 |
-
profile = "black"
|
10 |
-
line_length = 119
|
11 |
-
|
12 |
-
[tool.black]
|
13 |
-
line-length = 119
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
@@ -1,18 +1,2 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
click
|
4 |
-
datasets
|
5 |
-
gradio
|
6 |
-
gradio_client
|
7 |
-
huggingface-hub>=0.18.0
|
8 |
-
matplotlib
|
9 |
-
numpy
|
10 |
-
pandas
|
11 |
-
python-dateutil
|
12 |
-
requests
|
13 |
-
tqdm
|
14 |
-
transformers
|
15 |
-
tokenizers>=0.15.0
|
16 |
-
git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
|
17 |
-
accelerate
|
18 |
-
sentencepiece
|
|
|
1 |
+
gradio==4.36.1
|
2 |
+
pandas==2.2.2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/about.py
DELETED
@@ -1,72 +0,0 @@
|
|
1 |
-
from dataclasses import dataclass
|
2 |
-
from enum import Enum
|
3 |
-
|
4 |
-
@dataclass
|
5 |
-
class Task:
|
6 |
-
benchmark: str
|
7 |
-
metric: str
|
8 |
-
col_name: str
|
9 |
-
|
10 |
-
|
11 |
-
# Select your tasks here
|
12 |
-
# ---------------------------------------------------
|
13 |
-
class Tasks(Enum):
|
14 |
-
# task_key in the json file, metric_key in the json file, name to display in the leaderboard
|
15 |
-
task0 = Task("anli_r1", "acc", "ANLI")
|
16 |
-
task1 = Task("logiqa", "acc_norm", "LogiQA")
|
17 |
-
|
18 |
-
NUM_FEWSHOT = 0 # Change with your few shot
|
19 |
-
# ---------------------------------------------------
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
# Your leaderboard name
|
24 |
-
TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
|
25 |
-
|
26 |
-
# What does your leaderboard evaluate?
|
27 |
-
INTRODUCTION_TEXT = """
|
28 |
-
Intro text
|
29 |
-
"""
|
30 |
-
|
31 |
-
# Which evaluations are you running? how can people reproduce what you have?
|
32 |
-
LLM_BENCHMARKS_TEXT = f"""
|
33 |
-
## How it works
|
34 |
-
|
35 |
-
## Reproducibility
|
36 |
-
To reproduce our results, here is the commands you can run:
|
37 |
-
|
38 |
-
"""
|
39 |
-
|
40 |
-
EVALUATION_QUEUE_TEXT = """
|
41 |
-
## Some good practices before submitting a model
|
42 |
-
|
43 |
-
### 1) Make sure you can load your model and tokenizer using AutoClasses:
|
44 |
-
```python
|
45 |
-
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
46 |
-
config = AutoConfig.from_pretrained("your model name", revision=revision)
|
47 |
-
model = AutoModel.from_pretrained("your model name", revision=revision)
|
48 |
-
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
|
49 |
-
```
|
50 |
-
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
|
51 |
-
|
52 |
-
Note: make sure your model is public!
|
53 |
-
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
|
54 |
-
|
55 |
-
### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
|
56 |
-
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
|
57 |
-
|
58 |
-
### 3) Make sure your model has an open license!
|
59 |
-
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
|
60 |
-
|
61 |
-
### 4) Fill up your model card
|
62 |
-
When we add extra information about models to the leaderboard, it will be automatically taken from the model card
|
63 |
-
|
64 |
-
## In case of model failure
|
65 |
-
If your model is displayed in the `FAILED` category, its execution stopped.
|
66 |
-
Make sure you have followed the above steps first.
|
67 |
-
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
|
68 |
-
"""
|
69 |
-
|
70 |
-
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
|
71 |
-
CITATION_BUTTON_TEXT = r"""
|
72 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/display/css_html_js.py
DELETED
@@ -1,105 +0,0 @@
|
|
1 |
-
custom_css = """
|
2 |
-
|
3 |
-
.markdown-text {
|
4 |
-
font-size: 16px !important;
|
5 |
-
}
|
6 |
-
|
7 |
-
#models-to-add-text {
|
8 |
-
font-size: 18px !important;
|
9 |
-
}
|
10 |
-
|
11 |
-
#citation-button span {
|
12 |
-
font-size: 16px !important;
|
13 |
-
}
|
14 |
-
|
15 |
-
#citation-button textarea {
|
16 |
-
font-size: 16px !important;
|
17 |
-
}
|
18 |
-
|
19 |
-
#citation-button > label > button {
|
20 |
-
margin: 6px;
|
21 |
-
transform: scale(1.3);
|
22 |
-
}
|
23 |
-
|
24 |
-
#leaderboard-table {
|
25 |
-
margin-top: 15px
|
26 |
-
}
|
27 |
-
|
28 |
-
#leaderboard-table-lite {
|
29 |
-
margin-top: 15px
|
30 |
-
}
|
31 |
-
|
32 |
-
#search-bar-table-box > div:first-child {
|
33 |
-
background: none;
|
34 |
-
border: none;
|
35 |
-
}
|
36 |
-
|
37 |
-
#search-bar {
|
38 |
-
padding: 0px;
|
39 |
-
}
|
40 |
-
|
41 |
-
/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
|
42 |
-
table td:first-child,
|
43 |
-
table th:first-child {
|
44 |
-
max-width: 400px;
|
45 |
-
overflow: auto;
|
46 |
-
white-space: nowrap;
|
47 |
-
}
|
48 |
-
|
49 |
-
.tab-buttons button {
|
50 |
-
font-size: 20px;
|
51 |
-
}
|
52 |
-
|
53 |
-
#scale-logo {
|
54 |
-
border-style: none !important;
|
55 |
-
box-shadow: none;
|
56 |
-
display: block;
|
57 |
-
margin-left: auto;
|
58 |
-
margin-right: auto;
|
59 |
-
max-width: 600px;
|
60 |
-
}
|
61 |
-
|
62 |
-
#scale-logo .download {
|
63 |
-
display: none;
|
64 |
-
}
|
65 |
-
#filter_type{
|
66 |
-
border: 0;
|
67 |
-
padding-left: 0;
|
68 |
-
padding-top: 0;
|
69 |
-
}
|
70 |
-
#filter_type label {
|
71 |
-
display: flex;
|
72 |
-
}
|
73 |
-
#filter_type label > span{
|
74 |
-
margin-top: var(--spacing-lg);
|
75 |
-
margin-right: 0.5em;
|
76 |
-
}
|
77 |
-
#filter_type label > .wrap{
|
78 |
-
width: 103px;
|
79 |
-
}
|
80 |
-
#filter_type label > .wrap .wrap-inner{
|
81 |
-
padding: 2px;
|
82 |
-
}
|
83 |
-
#filter_type label > .wrap .wrap-inner input{
|
84 |
-
width: 1px
|
85 |
-
}
|
86 |
-
#filter-columns-type{
|
87 |
-
border:0;
|
88 |
-
padding:0.5;
|
89 |
-
}
|
90 |
-
#filter-columns-size{
|
91 |
-
border:0;
|
92 |
-
padding:0.5;
|
93 |
-
}
|
94 |
-
#box-filter > .form{
|
95 |
-
border: 0
|
96 |
-
}
|
97 |
-
"""
|
98 |
-
|
99 |
-
get_window_url_params = """
|
100 |
-
function(url_params) {
|
101 |
-
const params = new URLSearchParams(window.location.search);
|
102 |
-
url_params = Object.fromEntries(params);
|
103 |
-
return url_params;
|
104 |
-
}
|
105 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/display/formatting.py
DELETED
@@ -1,27 +0,0 @@
|
|
1 |
-
def model_hyperlink(link, model_name):
|
2 |
-
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
|
3 |
-
|
4 |
-
|
5 |
-
def make_clickable_model(model_name):
|
6 |
-
link = f"https://huggingface.co/{model_name}"
|
7 |
-
return model_hyperlink(link, model_name)
|
8 |
-
|
9 |
-
|
10 |
-
def styled_error(error):
|
11 |
-
return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
|
12 |
-
|
13 |
-
|
14 |
-
def styled_warning(warn):
|
15 |
-
return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
|
16 |
-
|
17 |
-
|
18 |
-
def styled_message(message):
|
19 |
-
return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
|
20 |
-
|
21 |
-
|
22 |
-
def has_no_nan_values(df, columns):
|
23 |
-
return df[columns].notna().all(axis=1)
|
24 |
-
|
25 |
-
|
26 |
-
def has_nan_values(df, columns):
|
27 |
-
return df[columns].isna().any(axis=1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/display/utils.py
DELETED
@@ -1,135 +0,0 @@
|
|
1 |
-
from dataclasses import dataclass, make_dataclass
|
2 |
-
from enum import Enum
|
3 |
-
|
4 |
-
import pandas as pd
|
5 |
-
|
6 |
-
from src.about import Tasks
|
7 |
-
|
8 |
-
def fields(raw_class):
|
9 |
-
return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
|
10 |
-
|
11 |
-
|
12 |
-
# These classes are for user facing column names,
|
13 |
-
# to avoid having to change them all around the code
|
14 |
-
# when a modif is needed
|
15 |
-
@dataclass
|
16 |
-
class ColumnContent:
|
17 |
-
name: str
|
18 |
-
type: str
|
19 |
-
displayed_by_default: bool
|
20 |
-
hidden: bool = False
|
21 |
-
never_hidden: bool = False
|
22 |
-
|
23 |
-
## Leaderboard columns
|
24 |
-
auto_eval_column_dict = []
|
25 |
-
# Init
|
26 |
-
auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
|
27 |
-
auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
|
28 |
-
#Scores
|
29 |
-
auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
|
30 |
-
for task in Tasks:
|
31 |
-
auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
|
32 |
-
# Model information
|
33 |
-
auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
|
34 |
-
auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
|
35 |
-
auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
|
36 |
-
auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
|
37 |
-
auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
|
38 |
-
auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
|
39 |
-
auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
|
40 |
-
auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
|
41 |
-
auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
|
42 |
-
|
43 |
-
# We use make dataclass to dynamically fill the scores from Tasks
|
44 |
-
AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
|
45 |
-
|
46 |
-
## For the queue columns in the submission tab
|
47 |
-
@dataclass(frozen=True)
|
48 |
-
class EvalQueueColumn: # Queue column
|
49 |
-
model = ColumnContent("model", "markdown", True)
|
50 |
-
revision = ColumnContent("revision", "str", True)
|
51 |
-
private = ColumnContent("private", "bool", True)
|
52 |
-
precision = ColumnContent("precision", "str", True)
|
53 |
-
weight_type = ColumnContent("weight_type", "str", "Original")
|
54 |
-
status = ColumnContent("status", "str", True)
|
55 |
-
|
56 |
-
## All the model information that we might need
|
57 |
-
@dataclass
|
58 |
-
class ModelDetails:
|
59 |
-
name: str
|
60 |
-
display_name: str = ""
|
61 |
-
symbol: str = "" # emoji
|
62 |
-
|
63 |
-
|
64 |
-
class ModelType(Enum):
|
65 |
-
PT = ModelDetails(name="pretrained", symbol="🟢")
|
66 |
-
FT = ModelDetails(name="fine-tuned", symbol="🔶")
|
67 |
-
IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
|
68 |
-
RL = ModelDetails(name="RL-tuned", symbol="🟦")
|
69 |
-
Unknown = ModelDetails(name="", symbol="?")
|
70 |
-
|
71 |
-
def to_str(self, separator=" "):
|
72 |
-
return f"{self.value.symbol}{separator}{self.value.name}"
|
73 |
-
|
74 |
-
@staticmethod
|
75 |
-
def from_str(type):
|
76 |
-
if "fine-tuned" in type or "🔶" in type:
|
77 |
-
return ModelType.FT
|
78 |
-
if "pretrained" in type or "🟢" in type:
|
79 |
-
return ModelType.PT
|
80 |
-
if "RL-tuned" in type or "🟦" in type:
|
81 |
-
return ModelType.RL
|
82 |
-
if "instruction-tuned" in type or "⭕" in type:
|
83 |
-
return ModelType.IFT
|
84 |
-
return ModelType.Unknown
|
85 |
-
|
86 |
-
class WeightType(Enum):
|
87 |
-
Adapter = ModelDetails("Adapter")
|
88 |
-
Original = ModelDetails("Original")
|
89 |
-
Delta = ModelDetails("Delta")
|
90 |
-
|
91 |
-
class Precision(Enum):
|
92 |
-
float16 = ModelDetails("float16")
|
93 |
-
bfloat16 = ModelDetails("bfloat16")
|
94 |
-
float32 = ModelDetails("float32")
|
95 |
-
#qt_8bit = ModelDetails("8bit")
|
96 |
-
#qt_4bit = ModelDetails("4bit")
|
97 |
-
#qt_GPTQ = ModelDetails("GPTQ")
|
98 |
-
Unknown = ModelDetails("?")
|
99 |
-
|
100 |
-
def from_str(precision):
|
101 |
-
if precision in ["torch.float16", "float16"]:
|
102 |
-
return Precision.float16
|
103 |
-
if precision in ["torch.bfloat16", "bfloat16"]:
|
104 |
-
return Precision.bfloat16
|
105 |
-
if precision in ["float32"]:
|
106 |
-
return Precision.float32
|
107 |
-
#if precision in ["8bit"]:
|
108 |
-
# return Precision.qt_8bit
|
109 |
-
#if precision in ["4bit"]:
|
110 |
-
# return Precision.qt_4bit
|
111 |
-
#if precision in ["GPTQ", "None"]:
|
112 |
-
# return Precision.qt_GPTQ
|
113 |
-
return Precision.Unknown
|
114 |
-
|
115 |
-
# Column selection
|
116 |
-
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
|
117 |
-
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
|
118 |
-
COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
|
119 |
-
TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
|
120 |
-
|
121 |
-
EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
|
122 |
-
EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
|
123 |
-
|
124 |
-
BENCHMARK_COLS = [t.value.col_name for t in Tasks]
|
125 |
-
|
126 |
-
NUMERIC_INTERVALS = {
|
127 |
-
"?": pd.Interval(-1, 0, closed="right"),
|
128 |
-
"~1.5": pd.Interval(0, 2, closed="right"),
|
129 |
-
"~3": pd.Interval(2, 4, closed="right"),
|
130 |
-
"~7": pd.Interval(4, 9, closed="right"),
|
131 |
-
"~13": pd.Interval(9, 20, closed="right"),
|
132 |
-
"~35": pd.Interval(20, 45, closed="right"),
|
133 |
-
"~60": pd.Interval(45, 70, closed="right"),
|
134 |
-
"70+": pd.Interval(70, 10000, closed="right"),
|
135 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/envs.py
DELETED
@@ -1,25 +0,0 @@
|
|
1 |
-
import os
|
2 |
-
|
3 |
-
from huggingface_hub import HfApi
|
4 |
-
|
5 |
-
# Info to change for your repository
|
6 |
-
# ----------------------------------
|
7 |
-
TOKEN = os.environ.get("TOKEN") # A read/write token for your org
|
8 |
-
|
9 |
-
OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
|
10 |
-
# ----------------------------------
|
11 |
-
|
12 |
-
REPO_ID = f"{OWNER}/leaderboard"
|
13 |
-
QUEUE_REPO = f"{OWNER}/requests"
|
14 |
-
RESULTS_REPO = f"{OWNER}/results"
|
15 |
-
|
16 |
-
# If you setup a cache later, just change HF_HOME
|
17 |
-
CACHE_PATH=os.getenv("HF_HOME", ".")
|
18 |
-
|
19 |
-
# Local caches
|
20 |
-
EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
|
21 |
-
EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
|
22 |
-
EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
|
23 |
-
EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
|
24 |
-
|
25 |
-
API = HfApi(token=TOKEN)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/leaderboard/read_evals.py
DELETED
@@ -1,196 +0,0 @@
|
|
1 |
-
import glob
|
2 |
-
import json
|
3 |
-
import math
|
4 |
-
import os
|
5 |
-
from dataclasses import dataclass
|
6 |
-
|
7 |
-
import dateutil
|
8 |
-
import numpy as np
|
9 |
-
|
10 |
-
from src.display.formatting import make_clickable_model
|
11 |
-
from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
|
12 |
-
from src.submission.check_validity import is_model_on_hub
|
13 |
-
|
14 |
-
|
15 |
-
@dataclass
|
16 |
-
class EvalResult:
|
17 |
-
"""Represents one full evaluation. Built from a combination of the result and request file for a given run.
|
18 |
-
"""
|
19 |
-
eval_name: str # org_model_precision (uid)
|
20 |
-
full_model: str # org/model (path on hub)
|
21 |
-
org: str
|
22 |
-
model: str
|
23 |
-
revision: str # commit hash, "" if main
|
24 |
-
results: dict
|
25 |
-
precision: Precision = Precision.Unknown
|
26 |
-
model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
|
27 |
-
weight_type: WeightType = WeightType.Original # Original or Adapter
|
28 |
-
architecture: str = "Unknown"
|
29 |
-
license: str = "?"
|
30 |
-
likes: int = 0
|
31 |
-
num_params: int = 0
|
32 |
-
date: str = "" # submission date of request file
|
33 |
-
still_on_hub: bool = False
|
34 |
-
|
35 |
-
@classmethod
|
36 |
-
def init_from_json_file(self, json_filepath):
|
37 |
-
"""Inits the result from the specific model result file"""
|
38 |
-
with open(json_filepath) as fp:
|
39 |
-
data = json.load(fp)
|
40 |
-
|
41 |
-
config = data.get("config")
|
42 |
-
|
43 |
-
# Precision
|
44 |
-
precision = Precision.from_str(config.get("model_dtype"))
|
45 |
-
|
46 |
-
# Get model and org
|
47 |
-
org_and_model = config.get("model_name", config.get("model_args", None))
|
48 |
-
org_and_model = org_and_model.split("/", 1)
|
49 |
-
|
50 |
-
if len(org_and_model) == 1:
|
51 |
-
org = None
|
52 |
-
model = org_and_model[0]
|
53 |
-
result_key = f"{model}_{precision.value.name}"
|
54 |
-
else:
|
55 |
-
org = org_and_model[0]
|
56 |
-
model = org_and_model[1]
|
57 |
-
result_key = f"{org}_{model}_{precision.value.name}"
|
58 |
-
full_model = "/".join(org_and_model)
|
59 |
-
|
60 |
-
still_on_hub, _, model_config = is_model_on_hub(
|
61 |
-
full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
|
62 |
-
)
|
63 |
-
architecture = "?"
|
64 |
-
if model_config is not None:
|
65 |
-
architectures = getattr(model_config, "architectures", None)
|
66 |
-
if architectures:
|
67 |
-
architecture = ";".join(architectures)
|
68 |
-
|
69 |
-
# Extract results available in this file (some results are split in several files)
|
70 |
-
results = {}
|
71 |
-
for task in Tasks:
|
72 |
-
task = task.value
|
73 |
-
|
74 |
-
# We average all scores of a given metric (not all metrics are present in all files)
|
75 |
-
accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
|
76 |
-
if accs.size == 0 or any([acc is None for acc in accs]):
|
77 |
-
continue
|
78 |
-
|
79 |
-
mean_acc = np.mean(accs) * 100.0
|
80 |
-
results[task.benchmark] = mean_acc
|
81 |
-
|
82 |
-
return self(
|
83 |
-
eval_name=result_key,
|
84 |
-
full_model=full_model,
|
85 |
-
org=org,
|
86 |
-
model=model,
|
87 |
-
results=results,
|
88 |
-
precision=precision,
|
89 |
-
revision= config.get("model_sha", ""),
|
90 |
-
still_on_hub=still_on_hub,
|
91 |
-
architecture=architecture
|
92 |
-
)
|
93 |
-
|
94 |
-
def update_with_request_file(self, requests_path):
|
95 |
-
"""Finds the relevant request file for the current model and updates info with it"""
|
96 |
-
request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
|
97 |
-
|
98 |
-
try:
|
99 |
-
with open(request_file, "r") as f:
|
100 |
-
request = json.load(f)
|
101 |
-
self.model_type = ModelType.from_str(request.get("model_type", ""))
|
102 |
-
self.weight_type = WeightType[request.get("weight_type", "Original")]
|
103 |
-
self.license = request.get("license", "?")
|
104 |
-
self.likes = request.get("likes", 0)
|
105 |
-
self.num_params = request.get("params", 0)
|
106 |
-
self.date = request.get("submitted_time", "")
|
107 |
-
except Exception:
|
108 |
-
print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
|
109 |
-
|
110 |
-
def to_dict(self):
|
111 |
-
"""Converts the Eval Result to a dict compatible with our dataframe display"""
|
112 |
-
average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
|
113 |
-
data_dict = {
|
114 |
-
"eval_name": self.eval_name, # not a column, just a save name,
|
115 |
-
AutoEvalColumn.precision.name: self.precision.value.name,
|
116 |
-
AutoEvalColumn.model_type.name: self.model_type.value.name,
|
117 |
-
AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
|
118 |
-
AutoEvalColumn.weight_type.name: self.weight_type.value.name,
|
119 |
-
AutoEvalColumn.architecture.name: self.architecture,
|
120 |
-
AutoEvalColumn.model.name: make_clickable_model(self.full_model),
|
121 |
-
AutoEvalColumn.revision.name: self.revision,
|
122 |
-
AutoEvalColumn.average.name: average,
|
123 |
-
AutoEvalColumn.license.name: self.license,
|
124 |
-
AutoEvalColumn.likes.name: self.likes,
|
125 |
-
AutoEvalColumn.params.name: self.num_params,
|
126 |
-
AutoEvalColumn.still_on_hub.name: self.still_on_hub,
|
127 |
-
}
|
128 |
-
|
129 |
-
for task in Tasks:
|
130 |
-
data_dict[task.value.col_name] = self.results[task.value.benchmark]
|
131 |
-
|
132 |
-
return data_dict
|
133 |
-
|
134 |
-
|
135 |
-
def get_request_file_for_model(requests_path, model_name, precision):
|
136 |
-
"""Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
|
137 |
-
request_files = os.path.join(
|
138 |
-
requests_path,
|
139 |
-
f"{model_name}_eval_request_*.json",
|
140 |
-
)
|
141 |
-
request_files = glob.glob(request_files)
|
142 |
-
|
143 |
-
# Select correct request file (precision)
|
144 |
-
request_file = ""
|
145 |
-
request_files = sorted(request_files, reverse=True)
|
146 |
-
for tmp_request_file in request_files:
|
147 |
-
with open(tmp_request_file, "r") as f:
|
148 |
-
req_content = json.load(f)
|
149 |
-
if (
|
150 |
-
req_content["status"] in ["FINISHED"]
|
151 |
-
and req_content["precision"] == precision.split(".")[-1]
|
152 |
-
):
|
153 |
-
request_file = tmp_request_file
|
154 |
-
return request_file
|
155 |
-
|
156 |
-
|
157 |
-
def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
|
158 |
-
"""From the path of the results folder root, extract all needed info for results"""
|
159 |
-
model_result_filepaths = []
|
160 |
-
|
161 |
-
for root, _, files in os.walk(results_path):
|
162 |
-
# We should only have json files in model results
|
163 |
-
if len(files) == 0 or any([not f.endswith(".json") for f in files]):
|
164 |
-
continue
|
165 |
-
|
166 |
-
# Sort the files by date
|
167 |
-
try:
|
168 |
-
files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
|
169 |
-
except dateutil.parser._parser.ParserError:
|
170 |
-
files = [files[-1]]
|
171 |
-
|
172 |
-
for file in files:
|
173 |
-
model_result_filepaths.append(os.path.join(root, file))
|
174 |
-
|
175 |
-
eval_results = {}
|
176 |
-
for model_result_filepath in model_result_filepaths:
|
177 |
-
# Creation of result
|
178 |
-
eval_result = EvalResult.init_from_json_file(model_result_filepath)
|
179 |
-
eval_result.update_with_request_file(requests_path)
|
180 |
-
|
181 |
-
# Store results of same eval together
|
182 |
-
eval_name = eval_result.eval_name
|
183 |
-
if eval_name in eval_results.keys():
|
184 |
-
eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
|
185 |
-
else:
|
186 |
-
eval_results[eval_name] = eval_result
|
187 |
-
|
188 |
-
results = []
|
189 |
-
for v in eval_results.values():
|
190 |
-
try:
|
191 |
-
v.to_dict() # we test if the dict version is complete
|
192 |
-
results.append(v)
|
193 |
-
except KeyError: # not all eval values present
|
194 |
-
continue
|
195 |
-
|
196 |
-
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/populate.py
DELETED
@@ -1,58 +0,0 @@
|
|
1 |
-
import json
|
2 |
-
import os
|
3 |
-
|
4 |
-
import pandas as pd
|
5 |
-
|
6 |
-
from src.display.formatting import has_no_nan_values, make_clickable_model
|
7 |
-
from src.display.utils import AutoEvalColumn, EvalQueueColumn
|
8 |
-
from src.leaderboard.read_evals import get_raw_eval_results
|
9 |
-
|
10 |
-
|
11 |
-
def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
|
12 |
-
"""Creates a dataframe from all the individual experiment results"""
|
13 |
-
raw_data = get_raw_eval_results(results_path, requests_path)
|
14 |
-
all_data_json = [v.to_dict() for v in raw_data]
|
15 |
-
|
16 |
-
df = pd.DataFrame.from_records(all_data_json)
|
17 |
-
df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
|
18 |
-
df = df[cols].round(decimals=2)
|
19 |
-
|
20 |
-
# filter out if any of the benchmarks have not been produced
|
21 |
-
df = df[has_no_nan_values(df, benchmark_cols)]
|
22 |
-
return raw_data, df
|
23 |
-
|
24 |
-
|
25 |
-
def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
|
26 |
-
"""Creates the different dataframes for the evaluation queues requestes"""
|
27 |
-
entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
|
28 |
-
all_evals = []
|
29 |
-
|
30 |
-
for entry in entries:
|
31 |
-
if ".json" in entry:
|
32 |
-
file_path = os.path.join(save_path, entry)
|
33 |
-
with open(file_path) as fp:
|
34 |
-
data = json.load(fp)
|
35 |
-
|
36 |
-
data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
|
37 |
-
data[EvalQueueColumn.revision.name] = data.get("revision", "main")
|
38 |
-
|
39 |
-
all_evals.append(data)
|
40 |
-
elif ".md" not in entry:
|
41 |
-
# this is a folder
|
42 |
-
sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
|
43 |
-
for sub_entry in sub_entries:
|
44 |
-
file_path = os.path.join(save_path, entry, sub_entry)
|
45 |
-
with open(file_path) as fp:
|
46 |
-
data = json.load(fp)
|
47 |
-
|
48 |
-
data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
|
49 |
-
data[EvalQueueColumn.revision.name] = data.get("revision", "main")
|
50 |
-
all_evals.append(data)
|
51 |
-
|
52 |
-
pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
|
53 |
-
running_list = [e for e in all_evals if e["status"] == "RUNNING"]
|
54 |
-
finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
|
55 |
-
df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
|
56 |
-
df_running = pd.DataFrame.from_records(running_list, columns=cols)
|
57 |
-
df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
|
58 |
-
return df_finished[cols], df_running[cols], df_pending[cols]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/submission/check_validity.py
DELETED
@@ -1,99 +0,0 @@
|
|
1 |
-
import json
|
2 |
-
import os
|
3 |
-
import re
|
4 |
-
from collections import defaultdict
|
5 |
-
from datetime import datetime, timedelta, timezone
|
6 |
-
|
7 |
-
import huggingface_hub
|
8 |
-
from huggingface_hub import ModelCard
|
9 |
-
from huggingface_hub.hf_api import ModelInfo
|
10 |
-
from transformers import AutoConfig
|
11 |
-
from transformers.models.auto.tokenization_auto import AutoTokenizer
|
12 |
-
|
13 |
-
def check_model_card(repo_id: str) -> tuple[bool, str]:
|
14 |
-
"""Checks if the model card and license exist and have been filled"""
|
15 |
-
try:
|
16 |
-
card = ModelCard.load(repo_id)
|
17 |
-
except huggingface_hub.utils.EntryNotFoundError:
|
18 |
-
return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
|
19 |
-
|
20 |
-
# Enforce license metadata
|
21 |
-
if card.data.license is None:
|
22 |
-
if not ("license_name" in card.data and "license_link" in card.data):
|
23 |
-
return False, (
|
24 |
-
"License not found. Please add a license to your model card using the `license` metadata or a"
|
25 |
-
" `license_name`/`license_link` pair."
|
26 |
-
)
|
27 |
-
|
28 |
-
# Enforce card content
|
29 |
-
if len(card.text) < 200:
|
30 |
-
return False, "Please add a description to your model card, it is too short."
|
31 |
-
|
32 |
-
return True, ""
|
33 |
-
|
34 |
-
def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
|
35 |
-
"""Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
|
36 |
-
try:
|
37 |
-
config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
38 |
-
if test_tokenizer:
|
39 |
-
try:
|
40 |
-
tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
|
41 |
-
except ValueError as e:
|
42 |
-
return (
|
43 |
-
False,
|
44 |
-
f"uses a tokenizer which is not in a transformers release: {e}",
|
45 |
-
None
|
46 |
-
)
|
47 |
-
except Exception as e:
|
48 |
-
return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
|
49 |
-
return True, None, config
|
50 |
-
|
51 |
-
except ValueError:
|
52 |
-
return (
|
53 |
-
False,
|
54 |
-
"needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
|
55 |
-
None
|
56 |
-
)
|
57 |
-
|
58 |
-
except Exception as e:
|
59 |
-
return False, "was not found on hub!", None
|
60 |
-
|
61 |
-
|
62 |
-
def get_model_size(model_info: ModelInfo, precision: str):
|
63 |
-
"""Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
|
64 |
-
try:
|
65 |
-
model_size = round(model_info.safetensors["total"] / 1e9, 3)
|
66 |
-
except (AttributeError, TypeError):
|
67 |
-
return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
|
68 |
-
|
69 |
-
size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
|
70 |
-
model_size = size_factor * model_size
|
71 |
-
return model_size
|
72 |
-
|
73 |
-
def get_model_arch(model_info: ModelInfo):
|
74 |
-
"""Gets the model architecture from the configuration"""
|
75 |
-
return model_info.config.get("architectures", "Unknown")
|
76 |
-
|
77 |
-
def already_submitted_models(requested_models_dir: str) -> set[str]:
|
78 |
-
"""Gather a list of already submitted models to avoid duplicates"""
|
79 |
-
depth = 1
|
80 |
-
file_names = []
|
81 |
-
users_to_submission_dates = defaultdict(list)
|
82 |
-
|
83 |
-
for root, _, files in os.walk(requested_models_dir):
|
84 |
-
current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
|
85 |
-
if current_depth == depth:
|
86 |
-
for file in files:
|
87 |
-
if not file.endswith(".json"):
|
88 |
-
continue
|
89 |
-
with open(os.path.join(root, file), "r") as f:
|
90 |
-
info = json.load(f)
|
91 |
-
file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
|
92 |
-
|
93 |
-
# Select organisation
|
94 |
-
if info["model"].count("/") == 0 or "submitted_time" not in info:
|
95 |
-
continue
|
96 |
-
organisation, _ = info["model"].split("/")
|
97 |
-
users_to_submission_dates[organisation].append(info["submitted_time"])
|
98 |
-
|
99 |
-
return set(file_names), users_to_submission_dates
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/submission/submit.py
DELETED
@@ -1,119 +0,0 @@
|
|
1 |
-
import json
|
2 |
-
import os
|
3 |
-
from datetime import datetime, timezone
|
4 |
-
|
5 |
-
from src.display.formatting import styled_error, styled_message, styled_warning
|
6 |
-
from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
|
7 |
-
from src.submission.check_validity import (
|
8 |
-
already_submitted_models,
|
9 |
-
check_model_card,
|
10 |
-
get_model_size,
|
11 |
-
is_model_on_hub,
|
12 |
-
)
|
13 |
-
|
14 |
-
REQUESTED_MODELS = None
|
15 |
-
USERS_TO_SUBMISSION_DATES = None
|
16 |
-
|
17 |
-
def add_new_eval(
|
18 |
-
model: str,
|
19 |
-
base_model: str,
|
20 |
-
revision: str,
|
21 |
-
precision: str,
|
22 |
-
weight_type: str,
|
23 |
-
model_type: str,
|
24 |
-
):
|
25 |
-
global REQUESTED_MODELS
|
26 |
-
global USERS_TO_SUBMISSION_DATES
|
27 |
-
if not REQUESTED_MODELS:
|
28 |
-
REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
|
29 |
-
|
30 |
-
user_name = ""
|
31 |
-
model_path = model
|
32 |
-
if "/" in model:
|
33 |
-
user_name = model.split("/")[0]
|
34 |
-
model_path = model.split("/")[1]
|
35 |
-
|
36 |
-
precision = precision.split(" ")[0]
|
37 |
-
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
38 |
-
|
39 |
-
if model_type is None or model_type == "":
|
40 |
-
return styled_error("Please select a model type.")
|
41 |
-
|
42 |
-
# Does the model actually exist?
|
43 |
-
if revision == "":
|
44 |
-
revision = "main"
|
45 |
-
|
46 |
-
# Is the model on the hub?
|
47 |
-
if weight_type in ["Delta", "Adapter"]:
|
48 |
-
base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
|
49 |
-
if not base_model_on_hub:
|
50 |
-
return styled_error(f'Base model "{base_model}" {error}')
|
51 |
-
|
52 |
-
if not weight_type == "Adapter":
|
53 |
-
model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
|
54 |
-
if not model_on_hub:
|
55 |
-
return styled_error(f'Model "{model}" {error}')
|
56 |
-
|
57 |
-
# Is the model info correctly filled?
|
58 |
-
try:
|
59 |
-
model_info = API.model_info(repo_id=model, revision=revision)
|
60 |
-
except Exception:
|
61 |
-
return styled_error("Could not get your model information. Please fill it up properly.")
|
62 |
-
|
63 |
-
model_size = get_model_size(model_info=model_info, precision=precision)
|
64 |
-
|
65 |
-
# Were the model card and license filled?
|
66 |
-
try:
|
67 |
-
license = model_info.cardData["license"]
|
68 |
-
except Exception:
|
69 |
-
return styled_error("Please select a license for your model")
|
70 |
-
|
71 |
-
modelcard_OK, error_msg = check_model_card(model)
|
72 |
-
if not modelcard_OK:
|
73 |
-
return styled_error(error_msg)
|
74 |
-
|
75 |
-
# Seems good, creating the eval
|
76 |
-
print("Adding new eval")
|
77 |
-
|
78 |
-
eval_entry = {
|
79 |
-
"model": model,
|
80 |
-
"base_model": base_model,
|
81 |
-
"revision": revision,
|
82 |
-
"precision": precision,
|
83 |
-
"weight_type": weight_type,
|
84 |
-
"status": "PENDING",
|
85 |
-
"submitted_time": current_time,
|
86 |
-
"model_type": model_type,
|
87 |
-
"likes": model_info.likes,
|
88 |
-
"params": model_size,
|
89 |
-
"license": license,
|
90 |
-
"private": False,
|
91 |
-
}
|
92 |
-
|
93 |
-
# Check for duplicate submission
|
94 |
-
if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
|
95 |
-
return styled_warning("This model has been already submitted.")
|
96 |
-
|
97 |
-
print("Creating eval file")
|
98 |
-
OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
|
99 |
-
os.makedirs(OUT_DIR, exist_ok=True)
|
100 |
-
out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
|
101 |
-
|
102 |
-
with open(out_path, "w") as f:
|
103 |
-
f.write(json.dumps(eval_entry))
|
104 |
-
|
105 |
-
print("Uploading eval file")
|
106 |
-
API.upload_file(
|
107 |
-
path_or_fileobj=out_path,
|
108 |
-
path_in_repo=out_path.split("eval-queue/")[1],
|
109 |
-
repo_id=QUEUE_REPO,
|
110 |
-
repo_type="dataset",
|
111 |
-
commit_message=f"Add {model} to eval queue",
|
112 |
-
)
|
113 |
-
|
114 |
-
# Remove the local file
|
115 |
-
os.remove(out_path)
|
116 |
-
|
117 |
-
return styled_message(
|
118 |
-
"Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
|
119 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|