{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import (\n",
    "    RobertaTokenizerFast,\n",
    "    RobertaForSequenceClassification,\n",
    "    TrainingArguments,\n",
    "    Trainer,\n",
    "    AutoConfig,\n",
    ")\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "folder_path = 'formatted_data/'\n",
    "\n",
    "# Get the list of all files in the folder\n",
    "file_names = os.listdir(folder_path)\n",
    "max_file_name=max([int(i.split(\"_\")[-1][:-4]) for i in file_names])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<>:2: SyntaxWarning: invalid escape sequence '\\d'\n",
      "<>:2: SyntaxWarning: invalid escape sequence '\\d'\n",
      "C:\\Users\\rajst\\AppData\\Local\\Temp\\ipykernel_18180\\3256903659.py:2: SyntaxWarning: invalid escape sequence '\\d'\n",
      "  df=pd.read_csv(\"formatted_data\\data_\"+str(max_file_name)+\".csv\")\n"
     ]
    }
   ],
   "source": [
    "device=\"cuda\"\n",
    "df=pd.read_csv(\"formatted_data\\data_\"+str(max_file_name)+\".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\external\\Experiments\\image_designing\\env\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n",
      "Some weights of LongformerForSequenceClassification were not initialized from the model checkpoint at allenai/longformer-base-4096 and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "import torch\n",
    "import pandas as pd\n",
    "\n",
    "# Assuming df is already defined and contains 'text' and 'label' columns\n",
    "# Convert labels to numerical format if they are not already (e.g., if they are strings)\n",
    "label_mapping = {label: idx for idx, label in enumerate(df['label'].unique())}\n",
    "df['label'] = df['label'].map(label_mapping)\n",
    "\n",
    "# Split the dataset into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(df['prompt'], df['label'], test_size=0.1, random_state=42)\n",
    "\n",
    "# Define the local directory to save the model\n",
    "local_model_dir = \"allenai/longformer-base-4096\"\n",
    "\n",
    "# Load the tokenizer and model from the local directory\n",
    "tokenizer = AutoTokenizer.from_pretrained(local_model_dir, add_prefix_space=True)\n",
    "num_labels = len(df['label'].unique())\n",
    "model = AutoModelForSequenceClassification.from_pretrained(local_model_dir, num_labels=num_labels)\n",
    "\n",
    "# Tokenize the data\n",
    "train_encodings = tokenizer(X_train.tolist(), truncation=True, padding=True, max_length=128)\n",
    "test_encodings = tokenizer(X_test.tolist(), truncation=True, padding=True, max_length=128)\n",
    "\n",
    "# Create a Dataset class\n",
    "class CustomDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        # Convert the encodings into tensors\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        # Add the labels as a tensor\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "# Create dataset objects\n",
    "train_dataset = CustomDataset(train_encodings, y_train.tolist())\n",
    "test_dataset = CustomDataset(test_encodings, y_test.tolist())\n",
    "\n",
    "# Check the structure of the first item for verification\n",
    "# print(train_dataset[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\external\\Experiments\\image_designing\\env\\Lib\\site-packages\\transformers\\training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n",
      "  0%|          | 0/3003 [00:00<?, ?it/s]Initializing global attention on CLS token...\n",
      "Input ids are automatically padded to be a multiple of `config.attention_window`: 512\n",
      "  0%|          | 10/3003 [00:30<1:46:54,  2.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.7056, 'grad_norm': 2.5160858631134033, 'learning_rate': 1.0000000000000002e-06, 'epoch': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 20/3003 [00:50<1:40:31,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6966, 'grad_norm': 1.4280675649642944, 'learning_rate': 2.0000000000000003e-06, 'epoch': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 30/3003 [01:10<1:39:47,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.7106, 'grad_norm': 1.0888476371765137, 'learning_rate': 3e-06, 'epoch': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▏         | 40/3003 [01:31<1:43:03,  2.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6912, 'grad_norm': 2.7551822662353516, 'learning_rate': 4.000000000000001e-06, 'epoch': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 50/3003 [01:52<1:39:39,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6678, 'grad_norm': 2.096587657928467, 'learning_rate': 5e-06, 'epoch': 0.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 60/3003 [02:13<1:39:18,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5825, 'grad_norm': 3.0920822620391846, 'learning_rate': 6e-06, 'epoch': 0.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 70/3003 [02:34<1:39:02,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4646, 'grad_norm': 3.9335849285125732, 'learning_rate': 7.000000000000001e-06, 'epoch': 0.02}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 80/3003 [02:55<1:38:27,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.214, 'grad_norm': 1.3212645053863525, 'learning_rate': 8.000000000000001e-06, 'epoch': 0.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 90/3003 [03:16<1:38:07,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0485, 'grad_norm': 0.46827349066734314, 'learning_rate': 9e-06, 'epoch': 0.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 100/3003 [03:36<1:37:59,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0276, 'grad_norm': 0.20443004369735718, 'learning_rate': 1e-05, 'epoch': 0.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▎         | 110/3003 [03:57<1:37:34,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0049, 'grad_norm': 0.07122860848903656, 'learning_rate': 1.1000000000000001e-05, 'epoch': 0.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 120/3003 [04:18<1:37:10,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0028, 'grad_norm': 0.04400772601366043, 'learning_rate': 1.2e-05, 'epoch': 0.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 130/3003 [04:39<1:36:56,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0029, 'grad_norm': 0.06917454302310944, 'learning_rate': 1.3000000000000001e-05, 'epoch': 0.04}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▍         | 140/3003 [04:59<1:35:06,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0014, 'grad_norm': 0.023964570835232735, 'learning_rate': 1.4000000000000001e-05, 'epoch': 0.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▍         | 150/3003 [05:20<1:34:39,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0011, 'grad_norm': 0.019553549587726593, 'learning_rate': 1.5e-05, 'epoch': 0.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 160/3003 [05:41<1:36:21,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0009, 'grad_norm': 0.018498791381716728, 'learning_rate': 1.6000000000000003e-05, 'epoch': 0.05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 170/3003 [06:01<1:35:39,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0008, 'grad_norm': 0.014466522261500359, 'learning_rate': 1.7000000000000003e-05, 'epoch': 0.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 180/3003 [06:22<1:35:21,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0007, 'grad_norm': 0.04788174107670784, 'learning_rate': 1.8e-05, 'epoch': 0.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▋         | 190/3003 [06:43<1:35:03,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0006, 'grad_norm': 0.015358459204435349, 'learning_rate': 1.9e-05, 'epoch': 0.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 200/3003 [07:04<1:34:39,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0005, 'grad_norm': 0.012000670656561852, 'learning_rate': 2e-05, 'epoch': 0.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 210/3003 [07:24<1:33:52,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0005, 'grad_norm': 0.012816125527024269, 'learning_rate': 2.1e-05, 'epoch': 0.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 220/3003 [07:45<1:33:52,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.009513739496469498, 'learning_rate': 2.2000000000000003e-05, 'epoch': 0.07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 230/3003 [08:06<1:33:37,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0006, 'grad_norm': 0.011809739284217358, 'learning_rate': 2.3000000000000003e-05, 'epoch': 0.08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 240/3003 [08:27<1:33:16,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0349, 'grad_norm': 0.011091427877545357, 'learning_rate': 2.4e-05, 'epoch': 0.08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 250/3003 [08:48<1:32:45,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0012, 'grad_norm': 0.014725973829627037, 'learning_rate': 2.5e-05, 'epoch': 0.08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▊         | 260/3003 [09:08<1:32:23,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.008555704727768898, 'learning_rate': 2.6000000000000002e-05, 'epoch': 0.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 270/3003 [09:29<1:32:10,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.005666612181812525, 'learning_rate': 2.7000000000000002e-05, 'epoch': 0.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 280/3003 [09:50<1:31:45,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0015, 'grad_norm': 0.006478630006313324, 'learning_rate': 2.8000000000000003e-05, 'epoch': 0.09}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|▉         | 290/3003 [10:11<1:31:36,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.01174585334956646, 'learning_rate': 2.9e-05, 'epoch': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|▉         | 300/3003 [10:32<1:31:36,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.005616085138171911, 'learning_rate': 3e-05, 'epoch': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 310/3003 [10:53<1:30:58,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.005002288613468409, 'learning_rate': 3.1e-05, 'epoch': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 320/3003 [11:14<1:30:33,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.007455029524862766, 'learning_rate': 3.2000000000000005e-05, 'epoch': 0.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 330/3003 [11:35<1:30:15,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.004513631574809551, 'learning_rate': 3.3e-05, 'epoch': 0.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█▏        | 340/3003 [11:55<1:29:57,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.00409993901848793, 'learning_rate': 3.4000000000000007e-05, 'epoch': 0.11}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 350/3003 [12:16<1:29:42,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0036373063921928406, 'learning_rate': 3.5e-05, 'epoch': 0.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 360/3003 [12:37<1:29:04,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.005301479250192642, 'learning_rate': 3.6e-05, 'epoch': 0.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 370/3003 [12:58<1:28:43,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0695, 'grad_norm': 0.1426796168088913, 'learning_rate': 3.7e-05, 'epoch': 0.12}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 380/3003 [13:19<1:28:19,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0476, 'grad_norm': 0.010606247931718826, 'learning_rate': 3.8e-05, 'epoch': 0.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 390/3003 [13:40<1:28:04,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.011695647612214088, 'learning_rate': 3.9000000000000006e-05, 'epoch': 0.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 400/3003 [14:00<1:27:42,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0070946794003248215, 'learning_rate': 4e-05, 'epoch': 0.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▎        | 410/3003 [14:21<1:27:20,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.006115735042840242, 'learning_rate': 4.1e-05, 'epoch': 0.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 420/3003 [14:42<1:27:04,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0031872543040663004, 'learning_rate': 4.2e-05, 'epoch': 0.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 430/3003 [15:03<1:26:37,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.009156874381005764, 'learning_rate': 4.3e-05, 'epoch': 0.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 440/3003 [15:23<1:25:29,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0033319571521133184, 'learning_rate': 4.4000000000000006e-05, 'epoch': 0.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 450/3003 [15:43<1:24:15,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0025260853581130505, 'learning_rate': 4.5e-05, 'epoch': 0.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 460/3003 [16:03<1:24:04,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0024102237075567245, 'learning_rate': 4.600000000000001e-05, 'epoch': 0.15}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 470/3003 [16:23<1:23:30,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0077, 'grad_norm': 0.007079532835632563, 'learning_rate': 4.7e-05, 'epoch': 0.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 480/3003 [16:44<1:23:41,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1249, 'grad_norm': 0.00524471141397953, 'learning_rate': 4.8e-05, 'epoch': 0.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▋        | 490/3003 [17:04<1:23:09,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0458, 'grad_norm': 0.017766565084457397, 'learning_rate': 4.9e-05, 'epoch': 0.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 500/3003 [17:25<1:23:03,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.002735314192250371, 'learning_rate': 5e-05, 'epoch': 0.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 510/3003 [18:41<1:51:03,  2.67s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.007025108207017183, 'learning_rate': 4.9800239712345186e-05, 'epoch': 0.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 520/3003 [19:01<1:22:32,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.003888959065079689, 'learning_rate': 4.960047942469037e-05, 'epoch': 0.17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 530/3003 [19:21<1:21:32,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016427605878561735, 'learning_rate': 4.940071913703556e-05, 'epoch': 0.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 540/3003 [19:42<1:21:25,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016121846856549382, 'learning_rate': 4.9200958849380744e-05, 'epoch': 0.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 550/3003 [20:02<1:21:23,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0576, 'grad_norm': 0.001291464432142675, 'learning_rate': 4.9001198561725934e-05, 'epoch': 0.18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▊        | 560/3003 [20:23<1:21:06,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.065, 'grad_norm': 0.0017117438837885857, 'learning_rate': 4.880143827407112e-05, 'epoch': 0.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 570/3003 [20:43<1:20:45,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0015254425816237926, 'learning_rate': 4.860167798641631e-05, 'epoch': 0.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 580/3003 [21:04<1:20:18,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0342, 'grad_norm': 0.024932725355029106, 'learning_rate': 4.840191769876149e-05, 'epoch': 0.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|█▉        | 590/3003 [21:24<1:19:52,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0031, 'grad_norm': 0.0019487085519358516, 'learning_rate': 4.8202157411106675e-05, 'epoch': 0.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|█▉        | 600/3003 [21:45<1:19:50,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001443441491574049, 'learning_rate': 4.800239712345186e-05, 'epoch': 0.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 610/3003 [22:05<1:19:11,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0013735247775912285, 'learning_rate': 4.780263683579704e-05, 'epoch': 0.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 620/3003 [22:26<1:19:00,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0012535795103758574, 'learning_rate': 4.760287654814223e-05, 'epoch': 0.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 630/3003 [22:46<1:18:34,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.007931673899292946, 'learning_rate': 4.7403116260487416e-05, 'epoch': 0.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██▏       | 640/3003 [23:06<1:18:15,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.001101399539038539, 'learning_rate': 4.72033559728326e-05, 'epoch': 0.21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 650/3003 [23:27<1:18:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009801016421988606, 'learning_rate': 4.700359568517779e-05, 'epoch': 0.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 660/3003 [23:47<1:17:52,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0585, 'grad_norm': 0.0682937428355217, 'learning_rate': 4.680383539752298e-05, 'epoch': 0.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 670/3003 [24:08<1:17:19,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.0015238254563882947, 'learning_rate': 4.6604075109868164e-05, 'epoch': 0.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 680/3003 [24:28<1:16:54,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0011254064738750458, 'learning_rate': 4.640431482221335e-05, 'epoch': 0.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 690/3003 [24:49<1:16:47,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008285316871479154, 'learning_rate': 4.620455453455853e-05, 'epoch': 0.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 700/3003 [25:09<1:16:27,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0453, 'grad_norm': 36.33842086791992, 'learning_rate': 4.6004794246903715e-05, 'epoch': 0.23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▎       | 710/3003 [25:30<1:16:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0019, 'grad_norm': 0.0009785413276404142, 'learning_rate': 4.5805033959248905e-05, 'epoch': 0.24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 720/3003 [25:50<1:15:40,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.0009566265507601202, 'learning_rate': 4.560527367159409e-05, 'epoch': 0.24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 730/3003 [26:11<1:15:21,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.001084203482605517, 'learning_rate': 4.540551338393927e-05, 'epoch': 0.24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▍       | 740/3003 [26:31<1:14:49,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008643826586194336, 'learning_rate': 4.5205753096284456e-05, 'epoch': 0.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▍       | 750/3003 [26:52<1:14:49,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0015566401416435838, 'learning_rate': 4.5005992808629646e-05, 'epoch': 0.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 760/3003 [27:12<1:14:24,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0006685077678412199, 'learning_rate': 4.4806232520974837e-05, 'epoch': 0.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 770/3003 [27:33<1:14:09,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008012795005925, 'learning_rate': 4.460647223332002e-05, 'epoch': 0.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 780/3003 [27:53<1:13:57,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0007247089524753392, 'learning_rate': 4.4406711945665204e-05, 'epoch': 0.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▋       | 790/3003 [28:14<1:13:28,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0007234471850097179, 'learning_rate': 4.420695165801039e-05, 'epoch': 0.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 800/3003 [28:34<1:13:06,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.000670120061840862, 'learning_rate': 4.400719137035558e-05, 'epoch': 0.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 810/3003 [28:54<1:12:38,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0005868024891242385, 'learning_rate': 4.380743108270076e-05, 'epoch': 0.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 820/3003 [29:14<1:12:05,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0006539876339957118, 'learning_rate': 4.3607670795045945e-05, 'epoch': 0.27}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 830/3003 [29:35<1:11:59,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0006512108375318348, 'learning_rate': 4.340791050739113e-05, 'epoch': 0.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 840/3003 [29:55<1:11:46,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0006434743409045041, 'learning_rate': 4.320815021973632e-05, 'epoch': 0.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 850/3003 [30:16<1:11:23,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0591, 'grad_norm': 0.0030941360164433718, 'learning_rate': 4.30083899320815e-05, 'epoch': 0.28}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▊       | 860/3003 [30:36<1:11:03,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1056, 'grad_norm': 0.008536402136087418, 'learning_rate': 4.280862964442669e-05, 'epoch': 0.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 870/3003 [30:57<1:10:51,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0011, 'grad_norm': 0.003404755610972643, 'learning_rate': 4.2608869356771876e-05, 'epoch': 0.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 880/3003 [31:17<1:10:27,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.016030175611376762, 'learning_rate': 4.240910906911706e-05, 'epoch': 0.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 890/3003 [31:38<1:10:06,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0435, 'grad_norm': 0.001489067799411714, 'learning_rate': 4.220934878146225e-05, 'epoch': 0.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 900/3003 [31:58<1:09:50,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0019906149245798588, 'learning_rate': 4.2009588493807434e-05, 'epoch': 0.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 910/3003 [32:19<1:09:25,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0011581499129533768, 'learning_rate': 4.180982820615262e-05, 'epoch': 0.3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 920/3003 [32:39<1:09:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008755039307288826, 'learning_rate': 4.16100679184978e-05, 'epoch': 0.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 930/3003 [33:00<1:08:48,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010396060533821583, 'learning_rate': 4.141030763084299e-05, 'epoch': 0.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███▏      | 940/3003 [33:20<1:08:18,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010882350616157055, 'learning_rate': 4.1210547343188175e-05, 'epoch': 0.31}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 950/3003 [33:41<1:08:07,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0652, 'grad_norm': 0.0022445744834840298, 'learning_rate': 4.101078705553336e-05, 'epoch': 0.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 960/3003 [34:01<1:07:47,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0469, 'grad_norm': 0.21104629337787628, 'learning_rate': 4.081102676787855e-05, 'epoch': 0.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 970/3003 [34:22<1:07:26,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1175, 'grad_norm': 0.009980568662285805, 'learning_rate': 4.061126648022373e-05, 'epoch': 0.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 980/3003 [34:42<1:07:07,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0249, 'grad_norm': 0.001976929372176528, 'learning_rate': 4.041150619256892e-05, 'epoch': 0.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 990/3003 [35:03<1:06:51,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.001363897230476141, 'learning_rate': 4.0211745904914107e-05, 'epoch': 0.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1000/3003 [35:23<1:06:28,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0012820296688005328, 'learning_rate': 4.001198561725929e-05, 'epoch': 0.33}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▎      | 1010/3003 [36:41<1:29:17,  2.69s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009462463785894215, 'learning_rate': 3.9812225329604474e-05, 'epoch': 0.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 1020/3003 [37:01<1:06:07,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.09, 'grad_norm': 0.0025044165086001158, 'learning_rate': 3.9612465041949664e-05, 'epoch': 0.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 1030/3003 [37:21<1:05:17,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0009, 'grad_norm': 0.02548227645456791, 'learning_rate': 3.941270475429485e-05, 'epoch': 0.34}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 1040/3003 [37:42<1:04:58,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.009632675908505917, 'learning_rate': 3.921294446664003e-05, 'epoch': 0.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 1050/3003 [38:02<1:04:35,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0023, 'grad_norm': 0.0027781990356743336, 'learning_rate': 3.9013184178985215e-05, 'epoch': 0.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 1060/3003 [38:22<1:04:22,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0008, 'grad_norm': 0.0032037850469350815, 'learning_rate': 3.8813423891330405e-05, 'epoch': 0.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 1070/3003 [38:43<1:03:59,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0519, 'grad_norm': 0.0018412364879623055, 'learning_rate': 3.8613663603675596e-05, 'epoch': 0.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 1080/3003 [39:03<1:03:43,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0872, 'grad_norm': 0.014253541827201843, 'learning_rate': 3.841390331602078e-05, 'epoch': 0.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▋      | 1090/3003 [39:24<1:03:17,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0033, 'grad_norm': 0.027335194870829582, 'learning_rate': 3.821414302836596e-05, 'epoch': 0.36}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 1100/3003 [39:44<1:03:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0546, 'grad_norm': 0.038219600915908813, 'learning_rate': 3.8014382740711146e-05, 'epoch': 0.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 1110/3003 [40:05<1:02:42,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0005, 'grad_norm': 0.013517574407160282, 'learning_rate': 3.781462245305634e-05, 'epoch': 0.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 1120/3003 [40:25<1:02:13,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0239, 'grad_norm': 0.006454634480178356, 'learning_rate': 3.761486216540152e-05, 'epoch': 0.37}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 1130/3003 [40:46<1:02:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.03900014981627464, 'learning_rate': 3.7415101877746704e-05, 'epoch': 0.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 1140/3003 [41:06<1:01:45,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0681, 'grad_norm': 0.00491346325725317, 'learning_rate': 3.721534159009189e-05, 'epoch': 0.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 1150/3003 [41:26<1:01:26,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0041275243274867535, 'learning_rate': 3.701558130243708e-05, 'epoch': 0.38}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▊      | 1160/3003 [41:47<1:01:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0027671516872942448, 'learning_rate': 3.681582101478226e-05, 'epoch': 0.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 1170/3003 [42:07<1:00:35,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002641943283379078, 'learning_rate': 3.661606072712745e-05, 'epoch': 0.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 1180/3003 [42:28<1:00:21,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0543, 'grad_norm': 39.52105712890625, 'learning_rate': 3.6416300439472635e-05, 'epoch': 0.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 1190/3003 [42:48<1:00:00,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.003473791992291808, 'learning_rate': 3.621654015181782e-05, 'epoch': 0.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 1200/3003 [43:09<59:44,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0027680464554578066, 'learning_rate': 3.601677986416301e-05, 'epoch': 0.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 1210/3003 [43:29<59:22,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.01078388374298811, 'learning_rate': 3.581701957650819e-05, 'epoch': 0.4}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 1220/3003 [43:50<59:03,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0357, 'grad_norm': 42.907527923583984, 'learning_rate': 3.5617259288853376e-05, 'epoch': 0.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 1230/3003 [44:10<58:46,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.004, 'grad_norm': 3.7802817821502686, 'learning_rate': 3.541749900119856e-05, 'epoch': 0.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████▏     | 1240/3003 [44:30<58:18,  1.98s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0014497467782348394, 'learning_rate': 3.5217738713543744e-05, 'epoch': 0.41}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 1250/3003 [44:51<57:59,  1.98s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015815888764336705, 'learning_rate': 3.5017978425888934e-05, 'epoch': 0.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 1260/3003 [45:11<57:46,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0314, 'grad_norm': 0.0015436438843607903, 'learning_rate': 3.4818218138234124e-05, 'epoch': 0.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 1270/3003 [45:32<57:25,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0014598466223105788, 'learning_rate': 3.461845785057931e-05, 'epoch': 0.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 1280/3003 [45:52<57:03,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0769, 'grad_norm': 0.04655052721500397, 'learning_rate': 3.441869756292449e-05, 'epoch': 0.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 1290/3003 [46:13<56:38,  1.98s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0007, 'grad_norm': 0.013303539715707302, 'learning_rate': 3.421893727526968e-05, 'epoch': 0.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 1300/3003 [46:33<56:23,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.008551524952054024, 'learning_rate': 3.4019176987614865e-05, 'epoch': 0.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▎     | 1310/3003 [46:53<56:04,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.008591330610215664, 'learning_rate': 3.381941669996005e-05, 'epoch': 0.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 1320/3003 [47:14<55:44,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.004096747376024723, 'learning_rate': 3.361965641230523e-05, 'epoch': 0.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 1330/3003 [47:34<55:28,  1.99s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0038160679396241903, 'learning_rate': 3.341989612465042e-05, 'epoch': 0.44}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 1340/3003 [47:55<55:00,  1.98s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0033866700250655413, 'learning_rate': 3.3220135836995607e-05, 'epoch': 0.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 1350/3003 [48:15<54:44,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0028249905444681644, 'learning_rate': 3.302037554934079e-05, 'epoch': 0.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 1360/3003 [48:36<54:30,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0023785040248185396, 'learning_rate': 3.282061526168598e-05, 'epoch': 0.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 1370/3003 [48:56<54:07,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002242030343040824, 'learning_rate': 3.2620854974031164e-05, 'epoch': 0.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 1380/3003 [49:17<53:48,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0024554606061428785, 'learning_rate': 3.2421094686376354e-05, 'epoch': 0.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▋     | 1390/3003 [49:37<53:21,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002487810095772147, 'learning_rate': 3.222133439872154e-05, 'epoch': 0.46}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 1400/3003 [49:58<53:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0022248649038374424, 'learning_rate': 3.202157411106672e-05, 'epoch': 0.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 1410/3003 [50:18<52:44,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016638458473607898, 'learning_rate': 3.1821813823411905e-05, 'epoch': 0.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 1420/3003 [50:38<52:17,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017281019827350974, 'learning_rate': 3.1622053535757096e-05, 'epoch': 0.47}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 1430/3003 [50:58<52:13,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017010862939059734, 'learning_rate': 3.142229324810228e-05, 'epoch': 0.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 1440/3003 [51:19<51:54,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0012866652105003595, 'learning_rate': 3.122253296044746e-05, 'epoch': 0.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 1450/3003 [51:39<51:36,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016131863230839372, 'learning_rate': 3.1022772672792646e-05, 'epoch': 0.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▊     | 1460/3003 [52:00<51:14,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0013754136161878705, 'learning_rate': 3.082301238513784e-05, 'epoch': 0.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 1470/3003 [52:20<50:51,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0010359800653532147, 'learning_rate': 3.062325209748303e-05, 'epoch': 0.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 1480/3003 [52:41<50:30,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009144794894382358, 'learning_rate': 3.0423491809828207e-05, 'epoch': 0.49}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|████▉     | 1490/3003 [53:01<50:11,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0012747239088639617, 'learning_rate': 3.0223731522173394e-05, 'epoch': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|████▉     | 1500/3003 [53:22<49:52,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0011199002619832754, 'learning_rate': 3.0023971234518578e-05, 'epoch': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1510/3003 [54:40<1:06:51,  2.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.001105917734093964, 'learning_rate': 2.9824210946863768e-05, 'epoch': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 1520/3003 [55:00<49:21,  2.00s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0011976523092016578, 'learning_rate': 2.9624450659208952e-05, 'epoch': 0.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 1530/3003 [55:20<48:49,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.001106030191294849, 'learning_rate': 2.9424690371554135e-05, 'epoch': 0.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████▏    | 1540/3003 [55:41<48:28,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008244980708695948, 'learning_rate': 2.9224930083899322e-05, 'epoch': 0.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 1550/3003 [56:01<48:15,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.001042555901221931, 'learning_rate': 2.9025169796244506e-05, 'epoch': 0.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 1560/3003 [56:22<47:50,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0526, 'grad_norm': 68.34154510498047, 'learning_rate': 2.8825409508589696e-05, 'epoch': 0.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 1570/3003 [56:42<47:30,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008120406419038773, 'learning_rate': 2.862564922093488e-05, 'epoch': 0.52}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 1580/3003 [57:03<47:11,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0011023794068023562, 'learning_rate': 2.8425888933280063e-05, 'epoch': 0.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 1590/3003 [57:23<46:53,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0006508004735223949, 'learning_rate': 2.822612864562525e-05, 'epoch': 0.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 1600/3003 [57:44<46:26,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009938995353877544, 'learning_rate': 2.802636835797044e-05, 'epoch': 0.53}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▎    | 1610/3003 [58:04<46:14,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009635526221245527, 'learning_rate': 2.7826608070315624e-05, 'epoch': 0.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 1620/3003 [58:25<45:47,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010660416446626186, 'learning_rate': 2.7626847782660808e-05, 'epoch': 0.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 1630/3003 [58:45<45:26,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.00101339106913656, 'learning_rate': 2.742708749500599e-05, 'epoch': 0.54}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 1640/3003 [59:06<45:08,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009901467710733414, 'learning_rate': 2.722732720735118e-05, 'epoch': 0.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 1650/3003 [59:26<44:51,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0011, 'grad_norm': 3.1940462589263916, 'learning_rate': 2.702756691969637e-05, 'epoch': 0.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 1660/3003 [59:47<44:41,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0092, 'grad_norm': 0.0007950253784656525, 'learning_rate': 2.6827806632041552e-05, 'epoch': 0.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 1670/3003 [1:00:07<44:14,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008654780685901642, 'learning_rate': 2.6628046344386736e-05, 'epoch': 0.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 1680/3003 [1:00:28<43:47,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.000814450962934643, 'learning_rate': 2.642828605673192e-05, 'epoch': 0.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▋    | 1690/3003 [1:00:48<43:32,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0418, 'grad_norm': 0.0006354842917062342, 'learning_rate': 2.622852576907711e-05, 'epoch': 0.56}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 1700/3003 [1:01:09<43:19,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0007463986985385418, 'learning_rate': 2.6028765481422297e-05, 'epoch': 0.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 1710/3003 [1:01:29<42:54,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008915060898289084, 'learning_rate': 2.582900519376748e-05, 'epoch': 0.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 1720/3003 [1:01:49<42:30,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0113, 'grad_norm': 0.0007307803607545793, 'learning_rate': 2.5629244906112664e-05, 'epoch': 0.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 1730/3003 [1:02:10<42:12,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.00083450524834916, 'learning_rate': 2.542948461845785e-05, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 1740/3003 [1:02:30<41:53,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0812, 'grad_norm': 0.0008523448486812413, 'learning_rate': 2.5229724330803038e-05, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 1750/3003 [1:02:51<41:33,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0007, 'grad_norm': 0.00469035841524601, 'learning_rate': 2.5029964043148225e-05, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▊    | 1760/3003 [1:03:11<41:14,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.3774644732475281, 'learning_rate': 2.483020375549341e-05, 'epoch': 0.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 1770/3003 [1:03:32<41:00,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002793041057884693, 'learning_rate': 2.4630443467838596e-05, 'epoch': 0.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 1780/3003 [1:03:52<40:33,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0008822784293442965, 'learning_rate': 2.443068318018378e-05, 'epoch': 0.59}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 1790/3003 [1:04:13<40:14,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010729030473157763, 'learning_rate': 2.4230922892528966e-05, 'epoch': 0.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 1800/3003 [1:04:33<39:54,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010390965035185218, 'learning_rate': 2.4031162604874153e-05, 'epoch': 0.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1810/3003 [1:04:54<39:36,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010601467220112681, 'learning_rate': 2.3831402317219337e-05, 'epoch': 0.6}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 1820/3003 [1:05:14<39:15,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0748, 'grad_norm': 0.0008922474808059633, 'learning_rate': 2.3631642029564524e-05, 'epoch': 0.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 1830/3003 [1:05:35<38:59,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0666, 'grad_norm': 0.003768255701288581, 'learning_rate': 2.3431881741909707e-05, 'epoch': 0.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████▏   | 1840/3003 [1:05:55<38:35,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002649817382916808, 'learning_rate': 2.3232121454254898e-05, 'epoch': 0.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 1850/3003 [1:06:16<38:14,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0019959695637226105, 'learning_rate': 2.303236116660008e-05, 'epoch': 0.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 1860/3003 [1:06:36<37:53,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0027332615572959185, 'learning_rate': 2.2832600878945268e-05, 'epoch': 0.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 1870/3003 [1:06:57<37:34,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0758, 'grad_norm': 0.00772722065448761, 'learning_rate': 2.2632840591290452e-05, 'epoch': 0.62}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 1880/3003 [1:07:17<37:12,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0039, 'grad_norm': 0.011659588664770126, 'learning_rate': 2.243308030363564e-05, 'epoch': 0.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 1890/3003 [1:07:38<36:57,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.0076921069994568825, 'learning_rate': 2.2233320015980826e-05, 'epoch': 0.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 1900/3003 [1:07:58<36:35,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.003882060991600156, 'learning_rate': 2.203355972832601e-05, 'epoch': 0.63}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▎   | 1910/3003 [1:08:19<36:10,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0026372068095952272, 'learning_rate': 2.1833799440671196e-05, 'epoch': 0.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 1920/3003 [1:08:39<35:53,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0028180652298033237, 'learning_rate': 2.163403915301638e-05, 'epoch': 0.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 1930/3003 [1:09:00<35:36,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0802, 'grad_norm': 0.002264478476718068, 'learning_rate': 2.1434278865361567e-05, 'epoch': 0.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 1940/3003 [1:09:20<35:12,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.003129880176857114, 'learning_rate': 2.1234518577706754e-05, 'epoch': 0.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 1950/3003 [1:09:40<34:55,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.003713118378072977, 'learning_rate': 2.103475829005194e-05, 'epoch': 0.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 1960/3003 [1:10:01<34:34,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002333930227905512, 'learning_rate': 2.0834998002397124e-05, 'epoch': 0.65}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 1970/3003 [1:10:21<34:13,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0020616943947970867, 'learning_rate': 2.063523771474231e-05, 'epoch': 0.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 1980/3003 [1:10:42<33:55,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0020499620586633682, 'learning_rate': 2.0435477427087495e-05, 'epoch': 0.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▋   | 1990/3003 [1:11:02<33:32,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0524, 'grad_norm': 0.003861739533022046, 'learning_rate': 2.0235717139432682e-05, 'epoch': 0.66}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2000/3003 [1:11:23<33:12,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.004459477961063385, 'learning_rate': 2.003595685177787e-05, 'epoch': 0.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2010/3003 [1:12:41<44:25,  2.68s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002151204040274024, 'learning_rate': 1.9836196564123052e-05, 'epoch': 0.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2020/3003 [1:13:00<32:42,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0279, 'grad_norm': 0.0018344199052080512, 'learning_rate': 1.963643627646824e-05, 'epoch': 0.67}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 2030/3003 [1:13:21<32:01,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0037, 'grad_norm': 0.7606645822525024, 'learning_rate': 1.9436675988813423e-05, 'epoch': 0.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 2040/3003 [1:13:41<31:52,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.001435951329767704, 'learning_rate': 1.923691570115861e-05, 'epoch': 0.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 2050/3003 [1:14:01<31:34,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001557422918267548, 'learning_rate': 1.9037155413503797e-05, 'epoch': 0.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▊   | 2060/3003 [1:14:22<31:13,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0027, 'grad_norm': 0.020819218829274178, 'learning_rate': 1.8837395125848984e-05, 'epoch': 0.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 2070/3003 [1:14:42<30:52,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016430005198344588, 'learning_rate': 1.8637634838194168e-05, 'epoch': 0.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 2080/3003 [1:15:03<30:32,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0034502027556300163, 'learning_rate': 1.843787455053935e-05, 'epoch': 0.69}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 2090/3003 [1:15:23<30:24,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0010857564629986882, 'learning_rate': 1.8238114262884538e-05, 'epoch': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 2100/3003 [1:15:44<30:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0013684701407328248, 'learning_rate': 1.8038353975229725e-05, 'epoch': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 2110/3003 [1:16:04<29:37,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.001054738531820476, 'learning_rate': 1.7838593687574912e-05, 'epoch': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 2120/3003 [1:16:25<29:17,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0010159454541280866, 'learning_rate': 1.7638833399920096e-05, 'epoch': 0.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 2130/3003 [1:16:45<28:59,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0013471405254676938, 'learning_rate': 1.7439073112265283e-05, 'epoch': 0.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████▏  | 2140/3003 [1:17:06<28:35,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0011137662222608924, 'learning_rate': 1.723931282461047e-05, 'epoch': 0.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 2150/3003 [1:17:26<28:22,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009327486041001976, 'learning_rate': 1.7039552536955657e-05, 'epoch': 0.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 2160/3003 [1:17:47<27:57,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'grad_norm': 0.0009590471163392067, 'learning_rate': 1.683979224930084e-05, 'epoch': 0.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 2170/3003 [1:18:07<27:42,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0756, 'grad_norm': 0.007833680137991905, 'learning_rate': 1.6640031961646024e-05, 'epoch': 0.72}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 2180/3003 [1:18:28<27:25,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0783, 'grad_norm': 0.003539525205269456, 'learning_rate': 1.644027167399121e-05, 'epoch': 0.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 2190/3003 [1:18:48<26:58,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.005967519246041775, 'learning_rate': 1.6240511386336398e-05, 'epoch': 0.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 2200/3003 [1:19:08<26:41,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0744, 'grad_norm': 0.04771209880709648, 'learning_rate': 1.6040751098681585e-05, 'epoch': 0.73}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 2210/3003 [1:19:29<26:16,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.004945298191159964, 'learning_rate': 1.5840990811026768e-05, 'epoch': 0.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 2220/3003 [1:19:49<26:00,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.003241043072193861, 'learning_rate': 1.5641230523371955e-05, 'epoch': 0.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 2230/3003 [1:20:10<25:38,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0028159564826637506, 'learning_rate': 1.544147023571714e-05, 'epoch': 0.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▍  | 2240/3003 [1:20:30<25:18,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.00239733443595469, 'learning_rate': 1.5241709948062327e-05, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▍  | 2250/3003 [1:20:50<24:56,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0020698511507362127, 'learning_rate': 1.5041949660407511e-05, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 2260/3003 [1:21:11<24:37,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0019798115827143192, 'learning_rate': 1.4842189372752696e-05, 'epoch': 0.75}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 2270/3003 [1:21:31<24:17,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0387, 'grad_norm': 102.4980239868164, 'learning_rate': 1.4642429085097883e-05, 'epoch': 0.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 2280/3003 [1:21:52<23:57,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017953937640413642, 'learning_rate': 1.4442668797443069e-05, 'epoch': 0.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▋  | 2290/3003 [1:22:12<23:37,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015997900627553463, 'learning_rate': 1.4242908509788256e-05, 'epoch': 0.76}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 2300/3003 [1:22:33<23:23,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017158088739961386, 'learning_rate': 1.4043148222133439e-05, 'epoch': 0.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 2310/3003 [1:22:53<23:04,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0849, 'grad_norm': 0.001947887009009719, 'learning_rate': 1.3843387934478628e-05, 'epoch': 0.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 2320/3003 [1:23:14<22:37,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.013087349943816662, 'learning_rate': 1.3643627646823811e-05, 'epoch': 0.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 2330/3003 [1:23:34<22:19,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0004, 'grad_norm': 0.0059995101764798164, 'learning_rate': 1.3443867359168998e-05, 'epoch': 0.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 2340/3003 [1:23:55<21:56,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.006965160369873047, 'learning_rate': 1.3244107071514184e-05, 'epoch': 0.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 2350/3003 [1:24:15<21:36,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0003, 'grad_norm': 0.00430301995947957, 'learning_rate': 1.3044346783859367e-05, 'epoch': 0.78}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▊  | 2360/3003 [1:24:35<21:17,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.004265023395419121, 'learning_rate': 1.2844586496204556e-05, 'epoch': 0.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 2370/3003 [1:24:56<21:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0584, 'grad_norm': 0.004185492638498545, 'learning_rate': 1.264482620854974e-05, 'epoch': 0.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 2380/3003 [1:25:17<20:37,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.0036239763721823692, 'learning_rate': 1.2445065920894926e-05, 'epoch': 0.79}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████▉  | 2390/3003 [1:25:37<20:23,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.00426241522654891, 'learning_rate': 1.2245305633240113e-05, 'epoch': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████▉  | 2400/3003 [1:25:58<20:00,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0033759018406271935, 'learning_rate': 1.2045545345585297e-05, 'epoch': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 2410/3003 [1:26:18<19:37,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.003131147241219878, 'learning_rate': 1.1845785057930484e-05, 'epoch': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 2420/3003 [1:26:38<19:17,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0020667659118771553, 'learning_rate': 1.164602477027567e-05, 'epoch': 0.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 2430/3003 [1:26:59<18:59,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.00238225725479424, 'learning_rate': 1.1446264482620856e-05, 'epoch': 0.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████▏ | 2440/3003 [1:27:19<18:40,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0029736612923443317, 'learning_rate': 1.1246504194966041e-05, 'epoch': 0.81}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 2450/3003 [1:27:40<18:21,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0019425321370363235, 'learning_rate': 1.1046743907311227e-05, 'epoch': 0.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 2460/3003 [1:28:00<17:58,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0022723667789250612, 'learning_rate': 1.0846983619656414e-05, 'epoch': 0.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 2470/3003 [1:28:21<17:43,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001850175322033465, 'learning_rate': 1.0647223332001599e-05, 'epoch': 0.82}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 2480/3003 [1:28:41<17:21,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015704033430665731, 'learning_rate': 1.0447463044346784e-05, 'epoch': 0.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 2490/3003 [1:29:02<16:58,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0018979765009135008, 'learning_rate': 1.024770275669197e-05, 'epoch': 0.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 2500/3003 [1:29:22<16:38,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001450559706427157, 'learning_rate': 1.0047942469037155e-05, 'epoch': 0.83}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▎ | 2510/3003 [1:30:40<22:03,  2.69s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0021633312571793795, 'learning_rate': 9.848182181382342e-06, 'epoch': 0.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 2520/3003 [1:31:00<16:06,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016824963968247175, 'learning_rate': 9.648421893727527e-06, 'epoch': 0.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 2530/3003 [1:31:20<15:36,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001511031063273549, 'learning_rate': 9.448661606072712e-06, 'epoch': 0.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▍ | 2540/3003 [1:31:40<15:21,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015909188659861684, 'learning_rate': 9.2489013184179e-06, 'epoch': 0.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▍ | 2550/3003 [1:32:01<15:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015774256316944957, 'learning_rate': 9.049141030763085e-06, 'epoch': 0.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 2560/3003 [1:32:21<14:42,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0018447356997057796, 'learning_rate': 8.849380743108272e-06, 'epoch': 0.85}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 2570/3003 [1:32:42<14:22,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017823990201577544, 'learning_rate': 8.649620455453457e-06, 'epoch': 0.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 2580/3003 [1:33:02<14:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0018155112629756331, 'learning_rate': 8.449860167798642e-06, 'epoch': 0.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 2590/3003 [1:33:23<13:43,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016266221646219492, 'learning_rate': 8.250099880143827e-06, 'epoch': 0.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 2600/3003 [1:33:43<13:22,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016281650168821216, 'learning_rate': 8.050339592489013e-06, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 2610/3003 [1:34:04<13:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0014282367192208767, 'learning_rate': 7.8505793048342e-06, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 2620/3003 [1:34:24<12:40,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001280998345464468, 'learning_rate': 7.650819017179385e-06, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 2630/3003 [1:34:45<12:21,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016243074787780643, 'learning_rate': 7.451058729524571e-06, 'epoch': 0.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 2640/3003 [1:35:05<12:04,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001510755391791463, 'learning_rate': 7.251298441869757e-06, 'epoch': 0.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 2650/3003 [1:35:26<11:44,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0014790110290050507, 'learning_rate': 7.0515381542149425e-06, 'epoch': 0.88}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▊ | 2660/3003 [1:35:46<11:25,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0826, 'grad_norm': 0.0013770590303465724, 'learning_rate': 6.851777866560129e-06, 'epoch': 0.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 2670/3003 [1:36:07<11:04,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.005627672653645277, 'learning_rate': 6.652017578905315e-06, 'epoch': 0.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 2680/3003 [1:36:27<10:42,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.007456621620804071, 'learning_rate': 6.452257291250499e-06, 'epoch': 0.89}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|████████▉ | 2690/3003 [1:36:48<10:22,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'grad_norm': 0.004230380989611149, 'learning_rate': 6.252497003595685e-06, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|████████▉ | 2700/3003 [1:37:08<10:01,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0029314004350453615, 'learning_rate': 6.052736715940871e-06, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 2710/3003 [1:37:29<09:43,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0024290999863296747, 'learning_rate': 5.852976428286057e-06, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 2720/3003 [1:37:49<09:22,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0034537550527602434, 'learning_rate': 5.653216140631243e-06, 'epoch': 0.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 2730/3003 [1:38:10<09:02,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0736, 'grad_norm': 19.2128963470459, 'learning_rate': 5.453455852976429e-06, 'epoch': 0.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 2740/3003 [1:38:30<08:43,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0018706999253481627, 'learning_rate': 5.253695565321614e-06, 'epoch': 0.91}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 2750/3003 [1:38:50<08:23,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.002080187201499939, 'learning_rate': 5.0539352776667996e-06, 'epoch': 0.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 2760/3003 [1:39:11<08:03,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016986202681437135, 'learning_rate': 4.854174990011986e-06, 'epoch': 0.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 2770/3003 [1:39:31<07:42,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0021813169587403536, 'learning_rate': 4.654414702357172e-06, 'epoch': 0.92}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 2780/3003 [1:39:52<07:24,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0021626597736030817, 'learning_rate': 4.454654414702358e-06, 'epoch': 0.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 2790/3003 [1:40:12<07:03,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016862043412402272, 'learning_rate': 4.254894127047543e-06, 'epoch': 0.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 2800/3003 [1:40:32<06:44,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0019441705662757158, 'learning_rate': 4.0551338393927285e-06, 'epoch': 0.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▎| 2810/3003 [1:40:53<06:22,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0224, 'grad_norm': 0.003379260189831257, 'learning_rate': 3.855373551737915e-06, 'epoch': 0.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 2820/3003 [1:41:13<06:04,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017461442621424794, 'learning_rate': 3.6556132640831007e-06, 'epoch': 0.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 2830/3003 [1:41:34<05:43,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.00190436199773103, 'learning_rate': 3.4558529764282864e-06, 'epoch': 0.94}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 2840/3003 [1:41:54<05:24,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0014752659481018782, 'learning_rate': 3.2560926887734717e-06, 'epoch': 0.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 2850/3003 [1:42:15<05:05,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015859573613852262, 'learning_rate': 3.056332401118658e-06, 'epoch': 0.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 2860/3003 [1:42:35<04:45,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001617018599063158, 'learning_rate': 2.8565721134638436e-06, 'epoch': 0.95}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 2870/3003 [1:42:56<04:25,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017322506755590439, 'learning_rate': 2.6568118258090293e-06, 'epoch': 0.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 2880/3003 [1:43:16<04:04,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017315457807853818, 'learning_rate': 2.457051538154215e-06, 'epoch': 0.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 2890/3003 [1:43:36<03:44,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.001833472866564989, 'learning_rate': 2.2572912504994007e-06, 'epoch': 0.96}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 2900/3003 [1:43:57<03:25,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017824253300204873, 'learning_rate': 2.0575309628445864e-06, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 2910/3003 [1:44:17<03:04,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016106388065963984, 'learning_rate': 1.8577706751897725e-06, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 2920/3003 [1:44:38<02:44,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0013282370055094361, 'learning_rate': 1.658010387534958e-06, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 2930/3003 [1:44:58<02:24,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016192757757380605, 'learning_rate': 1.458250099880144e-06, 'epoch': 0.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 2940/3003 [1:45:19<02:05,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0016772911185398698, 'learning_rate': 1.2584898122253298e-06, 'epoch': 0.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 2950/3003 [1:45:39<01:45,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0077, 'grad_norm': 0.0013132932363077998, 'learning_rate': 1.0587295245705155e-06, 'epoch': 0.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▊| 2960/3003 [1:46:00<01:25,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0017027498688548803, 'learning_rate': 8.589692369157012e-07, 'epoch': 0.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 2970/3003 [1:46:20<01:05,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.064, 'grad_norm': 0.0016665913863107562, 'learning_rate': 6.592089492608869e-07, 'epoch': 0.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 2980/3003 [1:46:40<00:45,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0015621671918779612, 'learning_rate': 4.594486616060727e-07, 'epoch': 0.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 2990/3003 [1:47:01<00:25,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0018298302311450243, 'learning_rate': 2.596883739512585e-07, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 3000/3003 [1:47:21<00:05,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'grad_norm': 0.0014699451858177781, 'learning_rate': 5.992808629644426e-08, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      "100%|██████████| 3003/3003 [1:52:09<00:00,  2.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.0015117675065994263, 'eval_accuracy': 0.9997502497502497, 'eval_runtime': 168.0335, 'eval_samples_per_second': 23.829, 'eval_steps_per_second': 1.988, 'epoch': 1.0}\n",
      "{'train_runtime': 6729.8702, 'train_samples_per_second': 5.355, 'train_steps_per_second': 0.446, 'train_loss': 0.02579331953068177, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3003, training_loss=0.02579331953068177, metrics={'train_runtime': 6729.8702, 'train_samples_per_second': 5.355, 'train_steps_per_second': 0.446, 'total_flos': 2958788084668416.0, 'train_loss': 0.02579331953068177, 'epoch': 1.0})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "output_dir=\"results/\",\n",
    "num_train_epochs=1,\n",
    "per_device_train_batch_size=12,\n",
    "per_device_eval_batch_size=12,\n",
    "warmup_steps=500,\n",
    "weight_decay=0.0001,\n",
    "logging_dir='./logs',\n",
    "logging_steps=10,\n",
    "evaluation_strategy=\"epoch\",\n",
    "\n",
    "\n",
    ")\n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "model=model.to(device),\n",
    "args=training_args,\n",
    "train_dataset=train_dataset,\n",
    "eval_dataset=test_dataset,\n",
    "compute_metrics=lambda p: {'accuracy': (p.predictions. argmax(-1) == p.label_ids).astype(float).mean().item()})\n",
    "\n",
    "\n",
    "\n",
    "trainer.train()\n",
    "\n",
    "# trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "folder_path = 'model_collection/'\n",
    "\n",
    "# Get the list of all files in the folder\n",
    "file_names = os.listdir(folder_path)\n",
    "max_file_name=max([int(i.split(\"_\")[-1]) for i in file_names]+[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('model_collection/model_4\\\\tokenizer_config.json',\n",
       " 'model_collection/model_4\\\\special_tokens_map.json',\n",
       " 'model_collection/model_4\\\\vocab.json',\n",
       " 'model_collection/model_4\\\\merges.txt',\n",
       " 'model_collection/model_4\\\\added_tokens.json',\n",
       " 'model_collection/model_4\\\\tokenizer.json')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_pretrained(\"model_collection/model_\"+str(max_file_name+1))\n",
    "tokenizer.save_pretrained(\"model_collection/model_\"+str(max_file_name+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model_collection/model_3/'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"model_collection/model_\"+str(max_file_name)+\"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"text-classification\", model = \"model_collection/model_\"+str(max_file_name+1)+\"/\")\n",
    "# classifier(\"create a painting a painting\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'LABEL_1', 'score': 0.9998732805252075}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier(\"generate something new\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\external\\Experiments\\image_designing\\env\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"text-classification\", model = \"model_collection/model_\"+str(3)+\"/\")\n",
    "# classifier(\"create a painting a painting\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'label': 'LABEL_0', 'score': 0.9999868869781494}]\n"
     ]
    }
   ],
   "source": [
    "print(classifier(\"show me photo of a forest\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}