fix order of request file vs request file list, to avoid resubmitting issues

app.py

@@ -11,7 +11,7 @@ from src.display.about import (
     LLM_BENCHMARKS_TEXT,
     TITLE,
 )
-from src.display.css_html_js import custom_css, get_window_url_params
+from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
     COLS,
@@ -26,6 +26,7 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
+from src.submission.check_validity import already_submitted_models
 from src.tools.collections import update_collections
 from src.tools.plots import (
     create_metric_plot_obj,

src/submission/submit.py

@@ -13,8 +13,8 @@ from src.submission.check_validity import (
     user_submission_permission,
 )
 
-requested_models, users_to_submission_dates = already_submitted_models(EVAL_REQUESTS_PATH)
-
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
     model: str,
@@ -25,6 +25,12 @@ def add_new_eval(
     weight_type: str,
     model_type: str,
 ):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
@@ -33,7 +39,7 @@ def add_new_eval(
 
     # Is the user rate limited?
     user_can_submit, error_msg = user_submission_permission(
-        model, users_to_submission_dates, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+        model, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
     )
     if not user_can_submit:
         return styled_error(error_msg)
@@ -99,15 +105,15 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
 
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in requested_models:
-        return styled_warning("This model has been already submitted.")
-
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))