import json import os from datetime import datetime, timezone from src.display.formatting import styled_error, styled_message from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO def add_new_eval(model: str, weight_type: str, gguf_filename=None): user_name = "" model_path = model if "/" in model: user_name = model.split("/")[0] model_path = model.split("/")[1] current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") # Is the model info correctly filled? try: model_info = API.model_info(repo_id=model, revision='main') except Exception: return styled_error("Could not get your model information.") if weight_type=="safetensors": if len(gguf_filename)!=0: return styled_error("GGUF filename should be empty when using safetensors.") # Seems good, creating the eval print("Adding new eval") eval_entry = { "model": model, "weight_type": weight_type, "gguf_filename": gguf_filename, "status": "PENDING", "submitted_time": current_time, } print("Creating eval file") OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}" os.makedirs(OUT_DIR, exist_ok=True) out_path = f"{OUT_DIR}/{model_path}_eval_request_{current_time}.json" with open(out_path, "w") as f: f.write(json.dumps(eval_entry)) print("Uploading eval file") API.upload_file( path_or_fileobj=out_path, path_in_repo=out_path.split("eval-queue/")[1], repo_id=QUEUE_REPO, repo_type="dataset", commit_message=f"Add {model} to eval queue", ) # Remove the local file os.remove(out_path) return styled_message( "Your request has been submitted to the evaluation queue!\nPlease wait for up to five minutes for the model to show in the PENDING list." )