open-llm-leaderboard/open_llm_leaderboard · Can we add model layers?

I want to be able to see model layers right here. Currently I have to select a part of the models, then use this script:

import requests
from collections import defaultdict
model_ids= [
"djuna/Q2.5-Veltha-14B-0.5",
"ehristoforu/fp4-14b-v1-fix",
"sthenno/tempesthenno-nuslerp-001",
#...and more...
]
# Base URL for Hugging Face
base_url = "https://huggingface.co/{}/raw/main/model.safetensors.index.json"
# Function to fetch and extract highest layer from the model's JSON data
def get_highest_layer(model_id):
    url = base_url.format(model_id)  # Construct the full URL
    
    response = requests.get(url)
    
    if response.status_code == 200:
        data = response.json()  # Parse the JSON response
        
        # Initialize a variable to track the highest layer number
        highest_layer = -1
        
        # Iterate through the keys in the 'weight_map' section
        for key in data.get('weight_map', {}):
            if key.startswith("model.layers."):
                try:
                    # Extract the layer number from the key
                    layer_num = int(key.split('.')[2])  # Extract 'xx' from 'model.layers.xx'
                    highest_layer = max(highest_layer, layer_num)
                except ValueError:
                    continue  # Skip if the layer number is not a valid integer
        
        return highest_layer
    else:
        print(f"Failed to fetch data for {model_id}: {response.status_code}")
        return None


# Dictionary to store the distribution of layers
layer_distribution = defaultdict(int)

# Iterate over each model ID
for model_id in model_ids:
    highest_layer = get_highest_layer(model_id)
    
    if highest_layer is not None:
        print(f"Model {model_id} has {highest_layer + 1} layers.")  # +1 because layer indexing starts from 0
        layer_distribution[highest_layer + 1] += 1  # Store distribution (number of models with x layers)

# Print the distribution of layers
print("\nLayer Distribution:")
for layers, count in sorted(layer_distribution.items()):
    print(f"{layers} layers: {count} models")

If they were already there that would be cool, also I can't select a hundred of them if I don't want to get 429, so if you could run this for each model with a 10-sec delay in your backend, and add those values that would be cool!