File size: 1,462 Bytes
291dd47 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
import os
# Get the list of checkpoints from the file system
checkpoints = sorted([f for f in os.listdir() if f.startswith("step-000")])
# Read the last uploaded checkpoint
try:
with open("checkpoint_uploaded.txt", "r") as log_file:
uploaded_checkpoints = log_file.read().splitlines()
last_uploaded = uploaded_checkpoints[-1] if uploaded_checkpoints else ""
except FileNotFoundError:
last_uploaded = ""
print("Last file uploaded", last_uploaded)
# Consider only files after the last uploaded checkpoint
if last_uploaded:
remaining_checkpoints = [ckpt for ckpt in checkpoints if ckpt > last_uploaded]
else:
remaining_checkpoints = checkpoints
print("Remaining files", remaining_checkpoints)
# Select 10 evenly spaced checkpoints from remaining ones
if remaining_checkpoints:
selected_checkpoints = [remaining_checkpoints[i] for i in range(0, len(remaining_checkpoints), max(1, len(remaining_checkpoints)//10))][:10]
else:
selected_checkpoints = []
print("Files to upload", selected_checkpoints)
# Upload each checkpoint using Hugging Face CLI and log the uploaded checkpoints
repo = "keeeeenw/MicroLlama2-checkpoints"
with open("checkpoint_uploaded.txt", "a") as log_file:
for checkpoint in selected_checkpoints:
command = ["huggingface-cli", "upload", repo, checkpoint, checkpoint]
print(f"Uploading {checkpoint}...")
os.system(" ".join(command))
log_file.write(checkpoint + "\n")
|