gpt-neo-125M-dutch / commit.sh
yhavinga's picture
Saving scripts, log and checkpoint at step 70000
f1818f3
raw
history blame
500 Bytes
#!/bin/bash
source ~/venv/bin/activate
while true
do
echo -n "Checking at .. "
date
UPDATED=`git status | grep flax_model | grep modified`
if [ ! -z "$UPDATED" ]
then
sleep 120
FILE=$(find . -name `ls -tR runs | grep events | head -n 1` | tail -n 1)
STEP=`tensorboard --load_fast=true --inspect --event_file=$FILE | grep last_step | awk '{print $2}'`
git add runs
git add flax_model.msgpack
git commit -m "Saving weights and logs step $STEP"
fi
sleep 60
done