| from datasets import load_dataset, DownloadMode | |
| import json | |
| import os | |
| from huggingface_hub import HfApi , hf_hub_download | |
| dataset_id = "SDSC/digiwild-dataset" | |
| token = os.getenv("HUGGINGFACE_TOKEN") | |
| # Initialize API client | |
| api = HfApi(token=token) | |
| # Load all metadata files | |
| files = api.list_repo_files(dataset_id, repo_type="dataset") | |
| json_files = [file for file in files if file.endswith(".json")] | |
| # Load the metadata compilation | |
| try: | |
| data_files = "data/train-00000-of-00001.parquet" | |
| metadata = load_dataset( | |
| dataset_id, | |
| data_files=data_files) | |
| # Add new json entries to dataset | |
| for file in json_files: | |
| file = hf_hub_download(repo_id=dataset_id, filename=file, repo_type="dataset") | |
| with open(file, "r") as f: | |
| new = json.load(f) | |
| if not(new["image_md5"] in metadata["train"]["image_md5"]): | |
| metadata["train"] = metadata["train"].add_item(new) | |
| except: | |
| metadata = load_dataset( | |
| dataset_id, | |
| data_files=json_files) | |
| metadata.push_to_hub(dataset_id, token=token) |