Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 3,297 Bytes
e295ac3 df66f6e e295ac3 df66f6e e295ac3 2a5f9fb e295ac3 2a5f9fb e295ac3 2a5f9fb 06acefd e295ac3 2a5f9fb 6b87e28 e295ac3 6b87e28 e295ac3 6b87e28 2a5f9fb e295ac3 6b87e28 2a5f9fb bab5ced 2a5f9fb 6b87e28 e295ac3 6b87e28 ecacc0f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
import pandas as pd
from huggingface_hub import add_collection_item, delete_collection_item, get_collection, update_collection_item
from huggingface_hub.utils._errors import HfHubHTTPError
from pandas import DataFrame
from src.display.utils import AutoEvalColumn, ModelType
from src.envs import H4_TOKEN, PATH_TO_COLLECTION
# Specific intervals for the collections
intervals = {
"1B": pd.Interval(0, 1.5, closed="right"),
"3B": pd.Interval(2.5, 3.5, closed="neither"),
"7B": pd.Interval(6, 8, closed="neither"),
"13B": pd.Interval(10, 14, closed="neither"),
"30B": pd.Interval(25, 35, closed="neither"),
"65B": pd.Interval(60, 70, closed="neither"),
}
def _filter_by_type_and_size(df, model_type, size_interval):
"""Filter DataFrame by model type and parameter size interval."""
type_emoji = model_type.value.symbol[0]
filtered_df = df[df[AutoEvalColumn.model_type_symbol.name] == type_emoji]
params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
mask = params_column.apply(lambda x: x in size_interval)
return filtered_df.loc[mask]
def _add_models_to_collection(collection, models, model_type, size):
"""Add best models to the collection and update positions."""
cur_len_collection = len(collection.items)
for ix, model in enumerate(models, start=1):
try:
collection = add_collection_item(
PATH_TO_COLLECTION,
item_id=model,
item_type="model",
exists_ok=True,
note=f"Best {model_type.to_str(' ')} model of around {size} on the leaderboard today!",
token=H4_TOKEN,
)
# Ensure position is correct if item was added
if len(collection.items) > cur_len_collection:
item_object_id = collection.items[-1].item_object_id
update_collection_item(collection_slug=PATH_TO_COLLECTION, item_object_id=item_object_id, position=ix)
cur_len_collection = len(collection.items)
break # assuming we only add the top model
except HfHubHTTPError:
continue
def update_collections(df: DataFrame):
"""Update collections by filtering and adding the best models."""
collection = get_collection(collection_slug=PATH_TO_COLLECTION, token=H4_TOKEN)
cur_best_models = []
for model_type in ModelType:
if not model_type.value.name:
continue
for size, interval in intervals.items():
filtered_df = _filter_by_type_and_size(df, model_type, interval)
best_models = list(
filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.fullname.name][:10]
)
print(model_type.value.symbol, size, best_models)
_add_models_to_collection(collection, best_models, model_type, size)
cur_best_models.extend(best_models)
# Cleanup
existing_models = {item.item_id for item in collection.items}
to_remove = existing_models - set(cur_best_models)
for item_id in to_remove:
try:
delete_collection_item(collection_slug=PATH_TO_COLLECTION, item_object_id=item_id, token=H4_TOKEN)
except HfHubHTTPError:
continue |