Model save

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: uf-mistral-it-sft-iopo-iter1-short
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/nlee28/lucky/runs/qeqjijzk)
+# uf-mistral-it-sft-iopo-iter1-short
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.4
+- Pytorch 2.1.2.post303
+- Datasets 2.18.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9995320542817033,
+    "total_flos": 0.0,
+    "train_loss": 0.9322090081954271,
+    "train_runtime": 16459.2443,
+    "train_samples": 34190,
+    "train_samples_per_second": 2.077,
+    "train_steps_per_second": 0.065
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.4"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c51815144db9aa12108b8e3956000c31dc4025036a8f322d7cfacac8f9adda8e
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cc5acd6615af362cde2f6d4ff10136669d22caa03c2f4eb1c7499135ad866e1
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:571f3329ed1c50bcc8f28d6df410f6e9b9132eda4b8caaed9028973f58ea3cdf
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Jul21_02-26-23_gpu-1/events.out.tfevents.1721496594.gpu-1.408495.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:707ed87701e589dc56e2798ffe356f569be10fe7bf57f397e277330ac77308f8
-size 96288
+oid sha256:d3bcb24af4f0d4247eb36891e9eebba1ace1d5369f9d69bc7f5a15bd6814e50a
+size 96642

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9995320542817033,
+    "total_flos": 0.0,
+    "train_loss": 0.9322090081954271,
+    "train_runtime": 16459.2443,
+    "train_samples": 34190,
+    "train_samples_per_second": 2.077,
+    "train_steps_per_second": 0.065
+}

trainer_state.json

@@ -0,0 +1,1950 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9995320542817033,
+  "eval_steps": 500,
+  "global_step": 1068,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009358914365933552,
+      "grad_norm": 35.0,
+      "learning_rate": 4.672897196261682e-08,
+      "log_odds_chosen": 0.028124790638685226,
+      "log_odds_ratio": -0.7122673988342285,
+      "logits/chosen": -2.177924633026123,
+      "logits/rejected": -2.1597542762756348,
+      "logps/chosen": -0.49789491295814514,
+      "logps/rejected": -0.5033541321754456,
+      "loss": 1.2,
+      "nll_loss": 1.1897943019866943,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.018717828731867104,
+      "grad_norm": 31.25,
+      "learning_rate": 9.345794392523364e-08,
+      "log_odds_chosen": 0.10793106257915497,
+      "log_odds_ratio": -0.6836065649986267,
+      "logits/chosen": -2.2072064876556396,
+      "logits/rejected": -2.1955819129943848,
+      "logps/chosen": -0.509109377861023,
+      "logps/rejected": -0.5578223466873169,
+      "loss": 1.2711,
+      "nll_loss": 1.2594499588012695,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.028076743097800654,
+      "grad_norm": 27.5,
+      "learning_rate": 1.4018691588785045e-07,
+      "log_odds_chosen": -0.06382028758525848,
+      "log_odds_ratio": -0.7615261077880859,
+      "logits/chosen": -2.1590800285339355,
+      "logits/rejected": -2.1389636993408203,
+      "logps/chosen": -0.5088254809379578,
+      "logps/rejected": -0.48718294501304626,
+      "loss": 1.2219,
+      "nll_loss": 1.2155386209487915,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.03743565746373421,
+      "grad_norm": 32.5,
+      "learning_rate": 1.8691588785046729e-07,
+      "log_odds_chosen": 0.027226638048887253,
+      "log_odds_ratio": -0.7158316969871521,
+      "logits/chosen": -2.2030715942382812,
+      "logits/rejected": -2.182612419128418,
+      "logps/chosen": -0.47975221276283264,
+      "logps/rejected": -0.4883168339729309,
+      "loss": 1.2153,
+      "nll_loss": 1.2491893768310547,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.04679457182966776,
+      "grad_norm": 25.0,
+      "learning_rate": 2.336448598130841e-07,
+      "log_odds_chosen": 0.07931121438741684,
+      "log_odds_ratio": -0.7042826414108276,
+      "logits/chosen": -2.1056649684906006,
+      "logits/rejected": -2.101743459701538,
+      "logps/chosen": -0.4837135672569275,
+      "logps/rejected": -0.524016261100769,
+      "loss": 1.1557,
+      "nll_loss": 1.152883529663086,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.05615348619560131,
+      "grad_norm": 28.625,
+      "learning_rate": 2.803738317757009e-07,
+      "log_odds_chosen": 0.07345812022686005,
+      "log_odds_ratio": -0.6917258501052856,
+      "logits/chosen": -2.18230938911438,
+      "logits/rejected": -2.1790032386779785,
+      "logps/chosen": -0.4614621698856354,
+      "logps/rejected": -0.4911852777004242,
+      "loss": 1.203,
+      "nll_loss": 1.1553771495819092,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.06551240056153486,
+      "grad_norm": 24.25,
+      "learning_rate": 3.271028037383177e-07,
+      "log_odds_chosen": -0.0254877507686615,
+      "log_odds_ratio": -0.7420052886009216,
+      "logits/chosen": -2.2310657501220703,
+      "logits/rejected": -2.2102017402648926,
+      "logps/chosen": -0.4837246537208557,
+      "logps/rejected": -0.48194044828414917,
+      "loss": 1.227,
+      "nll_loss": 1.2863795757293701,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.07487131492746842,
+      "grad_norm": 23.875,
+      "learning_rate": 3.7383177570093457e-07,
+      "log_odds_chosen": 0.10875405371189117,
+      "log_odds_ratio": -0.691827654838562,
+      "logits/chosen": -2.2158102989196777,
+      "logits/rejected": -2.198007345199585,
+      "logps/chosen": -0.48060980439186096,
+      "logps/rejected": -0.5459263324737549,
+      "loss": 1.113,
+      "nll_loss": 1.0872571468353271,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.08423022929340196,
+      "grad_norm": 19.75,
+      "learning_rate": 4.205607476635514e-07,
+      "log_odds_chosen": -0.004326423164457083,
+      "log_odds_ratio": -0.7243191003799438,
+      "logits/chosen": -2.2511162757873535,
+      "logits/rejected": -2.2321133613586426,
+      "logps/chosen": -0.49657949805259705,
+      "logps/rejected": -0.49786829948425293,
+      "loss": 1.1546,
+      "nll_loss": 1.1654746532440186,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.09358914365933552,
+      "grad_norm": 19.375,
+      "learning_rate": 4.672897196261682e-07,
+      "log_odds_chosen": 0.0023641125299036503,
+      "log_odds_ratio": -0.7194215059280396,
+      "logits/chosen": -2.1953084468841553,
+      "logits/rejected": -2.188227891921997,
+      "logps/chosen": -0.4719774127006531,
+      "logps/rejected": -0.46587830781936646,
+      "loss": 1.109,
+      "nll_loss": 1.142874836921692,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.10294805802526907,
+      "grad_norm": 14.8125,
+      "learning_rate": 4.999879772999679e-07,
+      "log_odds_chosen": 0.05830005556344986,
+      "log_odds_ratio": -0.6932491660118103,
+      "logits/chosen": -2.1755123138427734,
+      "logits/rejected": -2.1791396141052246,
+      "logps/chosen": -0.4585028290748596,
+      "logps/rejected": -0.48959556221961975,
+      "loss": 1.0518,
+      "nll_loss": 1.0515038967132568,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.11230697239120262,
+      "grad_norm": 13.5625,
+      "learning_rate": 4.997742725777528e-07,
+      "log_odds_chosen": 0.05845099687576294,
+      "log_odds_ratio": -0.692961573600769,
+      "logits/chosen": -2.2289061546325684,
+      "logits/rejected": -2.2106642723083496,
+      "logps/chosen": -0.47383299469947815,
+      "logps/rejected": -0.49634575843811035,
+      "loss": 1.005,
+      "nll_loss": 0.98981773853302,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.12166588675713617,
+      "grad_norm": 12.5,
+      "learning_rate": 4.992936596071119e-07,
+      "log_odds_chosen": 0.08409204334020615,
+      "log_odds_ratio": -0.6914924383163452,
+      "logits/chosen": -2.2284302711486816,
+      "logits/rejected": -2.2103445529937744,
+      "logps/chosen": -0.46478062868118286,
+      "logps/rejected": -0.5053035020828247,
+      "loss": 1.038,
+      "nll_loss": 1.0772876739501953,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.13102480112306972,
+      "grad_norm": 12.0625,
+      "learning_rate": 4.985466519700276e-07,
+      "log_odds_chosen": 0.05864203721284866,
+      "log_odds_ratio": -0.702600359916687,
+      "logits/chosen": -2.2445855140686035,
+      "logits/rejected": -2.23516845703125,
+      "logps/chosen": -0.47693437337875366,
+      "logps/rejected": -0.5074238777160645,
+      "loss": 1.0134,
+      "nll_loss": 1.022524118423462,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.14038371548900327,
+      "grad_norm": 12.625,
+      "learning_rate": 4.975340479172484e-07,
+      "log_odds_chosen": 0.02999536693096161,
+      "log_odds_ratio": -0.7136459350585938,
+      "logits/chosen": -2.2367188930511475,
+      "logits/rejected": -2.206267833709717,
+      "logps/chosen": -0.43298593163490295,
+      "logps/rejected": -0.44292062520980835,
+      "loss": 0.9101,
+      "nll_loss": 0.905490517616272,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.14974262985493683,
+      "grad_norm": 13.1875,
+      "learning_rate": 4.962569295152789e-07,
+      "log_odds_chosen": 0.11916179955005646,
+      "log_odds_ratio": -0.6726263165473938,
+      "logits/chosen": -2.1630289554595947,
+      "logits/rejected": -2.166447401046753,
+      "logps/chosen": -0.46223917603492737,
+      "logps/rejected": -0.49929705262184143,
+      "loss": 1.0266,
+      "nll_loss": 1.0065381526947021,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.1591015442208704,
+      "grad_norm": 12.3125,
+      "learning_rate": 4.947166614900862e-07,
+      "log_odds_chosen": 0.04568071290850639,
+      "log_odds_ratio": -0.7053205966949463,
+      "logits/chosen": -2.2215309143066406,
+      "logits/rejected": -2.2159583568573,
+      "logps/chosen": -0.46625766158103943,
+      "logps/rejected": -0.4844442903995514,
+      "loss": 0.9139,
+      "nll_loss": 0.9563388824462891,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.16846045858680392,
+      "grad_norm": 12.625,
+      "learning_rate": 4.929148897687566e-07,
+      "log_odds_chosen": 0.20167502760887146,
+      "log_odds_ratio": -0.688310980796814,
+      "logits/chosen": -2.220001697540283,
+      "logits/rejected": -2.2004828453063965,
+      "logps/chosen": -0.4466385841369629,
+      "logps/rejected": -0.5849505662918091,
+      "loss": 0.9175,
+      "nll_loss": 0.9583885073661804,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.17781937295273748,
+      "grad_norm": 11.0,
+      "learning_rate": 4.908535397206616e-07,
+      "log_odds_chosen": 0.12753407657146454,
+      "log_odds_ratio": -0.6645318269729614,
+      "logits/chosen": -2.1886050701141357,
+      "logits/rejected": -2.1912922859191895,
+      "logps/chosen": -0.4209999442100525,
+      "logps/rejected": -0.46723008155822754,
+      "loss": 0.9702,
+      "nll_loss": 0.9464343786239624,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.18717828731867103,
+      "grad_norm": 11.125,
+      "learning_rate": 4.885348141000122e-07,
+      "log_odds_chosen": 0.10208719968795776,
+      "log_odds_ratio": -0.6818111538887024,
+      "logits/chosen": -2.190870761871338,
+      "logits/rejected": -2.1749870777130127,
+      "logps/chosen": -0.4555496275424957,
+      "logps/rejected": -0.5055020451545715,
+      "loss": 0.9374,
+      "nll_loss": 0.9525081515312195,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.1965372016846046,
+      "grad_norm": 11.25,
+      "learning_rate": 4.859611906920007e-07,
+      "log_odds_chosen": 0.17131951451301575,
+      "log_odds_ratio": -0.6447469592094421,
+      "logits/chosen": -2.181173324584961,
+      "logits/rejected": -2.1704306602478027,
+      "logps/chosen": -0.4280853271484375,
+      "logps/rejected": -0.49025410413742065,
+      "loss": 0.9279,
+      "nll_loss": 0.9838709831237793,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.20589611605053815,
+      "grad_norm": 11.9375,
+      "learning_rate": 4.831354196650446e-07,
+      "log_odds_chosen": 0.12637579441070557,
+      "log_odds_ratio": -0.6647494435310364,
+      "logits/chosen": -2.209960460662842,
+      "logits/rejected": -2.190532922744751,
+      "logps/chosen": -0.4298935532569885,
+      "logps/rejected": -0.4736618399620056,
+      "loss": 0.929,
+      "nll_loss": 0.947419285774231,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.21525503041647168,
+      "grad_norm": 11.5,
+      "learning_rate": 4.800605206319624e-07,
+      "log_odds_chosen": 0.17757382988929749,
+      "log_odds_ratio": -0.6481081247329712,
+      "logits/chosen": -2.1775827407836914,
+      "logits/rejected": -2.1693122386932373,
+      "logps/chosen": -0.3929872512817383,
+      "logps/rejected": -0.4595734477043152,
+      "loss": 0.8857,
+      "nll_loss": 0.8631205558776855,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.22461394478240523,
+      "grad_norm": 12.1875,
+      "learning_rate": 4.767397794232225e-07,
+      "log_odds_chosen": 0.11623436212539673,
+      "log_odds_ratio": -0.6670342683792114,
+      "logits/chosen": -2.2236437797546387,
+      "logits/rejected": -2.198996067047119,
+      "logps/chosen": -0.39739638566970825,
+      "logps/rejected": -0.43490853905677795,
+      "loss": 0.8506,
+      "nll_loss": 0.8460358381271362,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.2339728591483388,
+      "grad_norm": 10.8125,
+      "learning_rate": 4.731767445757111e-07,
+      "log_odds_chosen": 0.1510472446680069,
+      "log_odds_ratio": -0.6724820137023926,
+      "logits/chosen": -2.1905274391174316,
+      "logits/rejected": -2.170478343963623,
+      "logps/chosen": -0.3974972665309906,
+      "logps/rejected": -0.46447643637657166,
+      "loss": 1.0051,
+      "nll_loss": 0.9697535634040833,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.24333177351427235,
+      "grad_norm": 10.9375,
+      "learning_rate": 4.6937522354077397e-07,
+      "log_odds_chosen": 0.034748904407024384,
+      "log_odds_ratio": -0.709030270576477,
+      "logits/chosen": -2.2467408180236816,
+      "logits/rejected": -2.23775315284729,
+      "logps/chosen": -0.41722503304481506,
+      "logps/rejected": -0.4315374791622162,
+      "loss": 0.9569,
+      "nll_loss": 0.9505971670150757,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.2526906878802059,
+      "grad_norm": 11.6875,
+      "learning_rate": 4.6533927861558166e-07,
+      "log_odds_chosen": 0.15775738656520844,
+      "log_odds_ratio": -0.6476392149925232,
+      "logits/chosen": -2.219115734100342,
+      "logits/rejected": -2.200603485107422,
+      "logps/chosen": -0.4007194936275482,
+      "logps/rejected": -0.45418375730514526,
+      "loss": 0.9836,
+      "nll_loss": 0.9320660829544067,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.26204960224613943,
+      "grad_norm": 13.5,
+      "learning_rate": 4.6107322260216787e-07,
+      "log_odds_chosen": 0.15421156585216522,
+      "log_odds_ratio": -0.6568454504013062,
+      "logits/chosen": -2.1702048778533936,
+      "logits/rejected": -2.15659499168396,
+      "logps/chosen": -0.38430947065353394,
+      "logps/rejected": -0.44216400384902954,
+      "loss": 0.8788,
+      "nll_loss": 0.8656591176986694,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.271408516612073,
+      "grad_norm": 12.4375,
+      "learning_rate": 4.565816141987782e-07,
+      "log_odds_chosen": 0.24107065796852112,
+      "log_odds_ratio": -0.6283782720565796,
+      "logits/chosen": -2.1911845207214355,
+      "logits/rejected": -2.170578718185425,
+      "logps/chosen": -0.38485488295555115,
+      "logps/rejected": -0.4777398109436035,
+      "loss": 0.9194,
+      "nll_loss": 0.9205155372619629,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.28076743097800655,
+      "grad_norm": 12.125,
+      "learning_rate": 4.518692531284555e-07,
+      "log_odds_chosen": 0.11865685135126114,
+      "log_odds_ratio": -0.6765984296798706,
+      "logits/chosen": -2.1895639896392822,
+      "logits/rejected": -2.1931369304656982,
+      "logps/chosen": -0.4192424714565277,
+      "logps/rejected": -0.46702760457992554,
+      "loss": 0.9343,
+      "nll_loss": 0.9279173612594604,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.2901263453439401,
+      "grad_norm": 12.0625,
+      "learning_rate": 4.469411750100657e-07,
+      "log_odds_chosen": 0.1528710126876831,
+      "log_odds_ratio": -0.6519570350646973,
+      "logits/chosen": -2.1230368614196777,
+      "logits/rejected": -2.111417293548584,
+      "logps/chosen": -0.41170787811279297,
+      "logps/rejected": -0.4662221372127533,
+      "loss": 0.9539,
+      "nll_loss": 0.9637632369995117,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.29948525970987366,
+      "grad_norm": 12.0,
+      "learning_rate": 4.418026459772465e-07,
+      "log_odds_chosen": 0.24052071571350098,
+      "log_odds_ratio": -0.6307070851325989,
+      "logits/chosen": -2.1706483364105225,
+      "logits/rejected": -2.166989803314209,
+      "logps/chosen": -0.3876957893371582,
+      "logps/rejected": -0.4977281093597412,
+      "loss": 0.9242,
+      "nll_loss": 0.8865762948989868,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.3088441740758072,
+      "grad_norm": 10.125,
+      "learning_rate": 4.3645915705102875e-07,
+      "log_odds_chosen": 0.12348359823226929,
+      "log_odds_ratio": -0.6693438291549683,
+      "logits/chosen": -2.2137680053710938,
+      "logits/rejected": -2.221555709838867,
+      "logps/chosen": -0.41634154319763184,
+      "logps/rejected": -0.46320948004722595,
+      "loss": 0.9491,
+      "nll_loss": 0.9716747403144836,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.3182030884417408,
+      "grad_norm": 10.125,
+      "learning_rate": 4.3091641827214255e-07,
+      "log_odds_chosen": 0.18769888579845428,
+      "log_odds_ratio": -0.6529528498649597,
+      "logits/chosen": -2.2090022563934326,
+      "logits/rejected": -2.2146294116973877,
+      "logps/chosen": -0.3794373869895935,
+      "logps/rejected": -0.45459240674972534,
+      "loss": 0.865,
+      "nll_loss": 0.8704261779785156,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.32756200280767434,
+      "grad_norm": 11.125,
+      "learning_rate": 4.251803525992806e-07,
+      "log_odds_chosen": 0.11162462085485458,
+      "log_odds_ratio": -0.6723114252090454,
+      "logits/chosen": -2.1960465908050537,
+      "logits/rejected": -2.1942174434661865,
+      "logps/chosen": -0.4046311378479004,
+      "logps/rejected": -0.452523797750473,
+      "loss": 0.8887,
+      "nll_loss": 0.9013251066207886,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.33692091717360784,
+      "grad_norm": 11.5625,
+      "learning_rate": 4.192570895798369e-07,
+      "log_odds_chosen": 0.18315447866916656,
+      "log_odds_ratio": -0.6423401832580566,
+      "logits/chosen": -2.184838056564331,
+      "logits/rejected": -2.1885063648223877,
+      "logps/chosen": -0.39569249749183655,
+      "logps/rejected": -0.4560003876686096,
+      "loss": 0.9208,
+      "nll_loss": 0.8708004951477051,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.3462798315395414,
+      "grad_norm": 12.1875,
+      "learning_rate": 4.1315295879988603e-07,
+      "log_odds_chosen": 0.13799132406711578,
+      "log_odds_ratio": -0.6605507731437683,
+      "logits/chosen": -2.1906182765960693,
+      "logits/rejected": -2.183380603790283,
+      "logps/chosen": -0.3925025463104248,
+      "logps/rejected": -0.4469067454338074,
+      "loss": 0.8882,
+      "nll_loss": 0.89342200756073,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.35563874590547495,
+      "grad_norm": 12.4375,
+      "learning_rate": 4.06874483120401e-07,
+      "log_odds_chosen": 0.18038353323936462,
+      "log_odds_ratio": -0.6446860432624817,
+      "logits/chosen": -2.227576494216919,
+      "logits/rejected": -2.209685802459717,
+      "logps/chosen": -0.3906407356262207,
+      "logps/rejected": -0.45715445280075073,
+      "loss": 0.9863,
+      "nll_loss": 0.9728641510009766,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.3649976602714085,
+      "grad_norm": 12.6875,
+      "learning_rate": 4.00428371706938e-07,
+      "log_odds_chosen": 0.16080859303474426,
+      "log_odds_ratio": -0.6434266567230225,
+      "logits/chosen": -2.237511396408081,
+      "logits/rejected": -2.213977336883545,
+      "logps/chosen": -0.39901459217071533,
+      "logps/rejected": -0.461778461933136,
+      "loss": 0.8935,
+      "nll_loss": 0.9344221353530884,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.37435657463734207,
+      "grad_norm": 11.875,
+      "learning_rate": 3.9382151286023644e-07,
+      "log_odds_chosen": 0.13357871770858765,
+      "log_odds_ratio": -0.6608735918998718,
+      "logits/chosen": -2.2457404136657715,
+      "logits/rejected": -2.2428553104400635,
+      "logps/chosen": -0.39655953645706177,
+      "logps/rejected": -0.4421748220920563,
+      "loss": 0.8421,
+      "nll_loss": 0.8112044334411621,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.3837154890032756,
+      "grad_norm": 12.375,
+      "learning_rate": 3.8706096665539617e-07,
+      "log_odds_chosen": 0.2025759220123291,
+      "log_odds_ratio": -0.6549407839775085,
+      "logits/chosen": -2.231397867202759,
+      "logits/rejected": -2.211425304412842,
+      "logps/chosen": -0.39163392782211304,
+      "logps/rejected": -0.4965929090976715,
+      "loss": 0.9318,
+      "nll_loss": 0.9396621584892273,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.3930744033692092,
+      "grad_norm": 11.0625,
+      "learning_rate": 3.801539573974959e-07,
+      "log_odds_chosen": 0.30672961473464966,
+      "log_odds_ratio": -0.6271880269050598,
+      "logits/chosen": -2.259282112121582,
+      "logits/rejected": -2.2307496070861816,
+      "logps/chosen": -0.3660631775856018,
+      "logps/rejected": -0.5345751047134399,
+      "loss": 0.91,
+      "nll_loss": 0.9085640907287598,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.40243331773514274,
+      "grad_norm": 10.125,
+      "learning_rate": 3.7310786590171683e-07,
+      "log_odds_chosen": 0.12312289327383041,
+      "log_odds_ratio": -0.666767418384552,
+      "logits/chosen": -2.2531981468200684,
+      "logits/rejected": -2.2322800159454346,
+      "logps/chosen": -0.4015491008758545,
+      "logps/rejected": -0.4476936459541321,
+      "loss": 0.9478,
+      "nll_loss": 0.9238454699516296,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.4117922321010763,
+      "grad_norm": 12.875,
+      "learning_rate": 3.659302216062191e-07,
+      "log_odds_chosen": 0.18517343699932098,
+      "log_odds_ratio": -0.6397743225097656,
+      "logits/chosen": -2.199251890182495,
+      "logits/rejected": -2.1922011375427246,
+      "logps/chosen": -0.38667598366737366,
+      "logps/rejected": -0.4541630148887634,
+      "loss": 0.899,
+      "nll_loss": 0.9328523874282837,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.42115114646700985,
+      "grad_norm": 10.5,
+      "learning_rate": 3.586286945262007e-07,
+      "log_odds_chosen": 0.19681106507778168,
+      "log_odds_ratio": -0.632773756980896,
+      "logits/chosen": -2.0825257301330566,
+      "logits/rejected": -2.0917224884033203,
+      "logps/chosen": -0.3770086467266083,
+      "logps/rejected": -0.4443152844905853,
+      "loss": 0.863,
+      "nll_loss": 0.8851884007453918,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.43051006083294335,
+      "grad_norm": 11.375,
+      "learning_rate": 3.512110870577357e-07,
+      "log_odds_chosen": 0.18732145428657532,
+      "log_odds_ratio": -0.6455721855163574,
+      "logits/chosen": -2.168400287628174,
+      "logits/rejected": -2.1601405143737793,
+      "logps/chosen": -0.3884671926498413,
+      "logps/rejected": -0.45501670241355896,
+      "loss": 0.958,
+      "nll_loss": 0.9003719091415405,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.4398689751988769,
+      "grad_norm": 9.0,
+      "learning_rate": 3.436853256401506e-07,
+      "log_odds_chosen": 0.18461090326309204,
+      "log_odds_ratio": -0.6388983726501465,
+      "logits/chosen": -2.175309896469116,
+      "logits/rejected": -2.1828722953796387,
+      "logps/chosen": -0.3819858431816101,
+      "logps/rejected": -0.4427759647369385,
+      "loss": 0.8924,
+      "nll_loss": 0.861772358417511,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.44922788956481047,
+      "grad_norm": 11.625,
+      "learning_rate": 3.3605945228584866e-07,
+      "log_odds_chosen": 0.14212632179260254,
+      "log_odds_ratio": -0.6561137437820435,
+      "logits/chosen": -2.226238965988159,
+      "logits/rejected": -2.214329242706299,
+      "logps/chosen": -0.39799243211746216,
+      "logps/rejected": -0.4503845274448395,
+      "loss": 0.9181,
+      "nll_loss": 0.9322090148925781,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.458586803930744,
+      "grad_norm": 10.5625,
+      "learning_rate": 3.283416159866321e-07,
+      "log_odds_chosen": 0.19436636567115784,
+      "log_odds_ratio": -0.6323596239089966,
+      "logits/chosen": -2.2073452472686768,
+      "logits/rejected": -2.2207274436950684,
+      "logps/chosen": -0.3859022557735443,
+      "logps/rejected": -0.45602479577064514,
+      "loss": 0.8872,
+      "nll_loss": 0.866982102394104,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.4679457182966776,
+      "grad_norm": 12.875,
+      "learning_rate": 3.205400640057081e-07,
+      "log_odds_chosen": 0.07180626690387726,
+      "log_odds_ratio": -0.7025886178016663,
+      "logits/chosen": -2.2078473567962646,
+      "logits/rejected": -2.205059289932251,
+      "logps/chosen": -0.39877811074256897,
+      "logps/rejected": -0.42040663957595825,
+      "loss": 0.9405,
+      "nll_loss": 0.9519411325454712,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.47730463266261114,
+      "grad_norm": 11.625,
+      "learning_rate": 3.126631330646801e-07,
+      "log_odds_chosen": 0.21832695603370667,
+      "log_odds_ratio": -0.6433119177818298,
+      "logits/chosen": -2.243464946746826,
+      "logits/rejected": -2.245924234390259,
+      "logps/chosen": -0.3781166076660156,
+      "logps/rejected": -0.46226248145103455,
+      "loss": 0.8928,
+      "nll_loss": 0.8886586427688599,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.4866635470285447,
+      "grad_norm": 11.0625,
+      "learning_rate": 3.0471924043494595e-07,
+      "log_odds_chosen": 0.1995456963777542,
+      "log_odds_ratio": -0.6375859975814819,
+      "logits/chosen": -2.1644656658172607,
+      "logits/rejected": -2.1902050971984863,
+      "logps/chosen": -0.38845792412757874,
+      "logps/rejected": -0.46764081716537476,
+      "loss": 0.8949,
+      "nll_loss": 0.872398853302002,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.49602246139447825,
+      "grad_norm": 10.0625,
+      "learning_rate": 2.967168749430191e-07,
+      "log_odds_chosen": 0.1926880180835724,
+      "log_odds_ratio": -0.6435622572898865,
+      "logits/chosen": -2.206993818283081,
+      "logits/rejected": -2.191556215286255,
+      "logps/chosen": -0.3731563985347748,
+      "logps/rejected": -0.45363467931747437,
+      "loss": 0.8595,
+      "nll_loss": 0.8209236264228821,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.5053813757604118,
+      "grad_norm": 10.9375,
+      "learning_rate": 2.8866458789938774e-07,
+      "log_odds_chosen": 0.25284868478775024,
+      "log_odds_ratio": -0.6184083819389343,
+      "logits/chosen": -2.21002197265625,
+      "logits/rejected": -2.218122720718384,
+      "logps/chosen": -0.3761533200740814,
+      "logps/rejected": -0.4766814708709717,
+      "loss": 0.8866,
+      "nll_loss": 0.8908084034919739,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.5147402901263454,
+      "grad_norm": 10.5,
+      "learning_rate": 2.8057098396060196e-07,
+      "log_odds_chosen": 0.07347196340560913,
+      "log_odds_ratio": -0.6952499747276306,
+      "logits/chosen": -2.2205636501312256,
+      "logits/rejected": -2.2049291133880615,
+      "logps/chosen": -0.39118486642837524,
+      "logps/rejected": -0.4140162467956543,
+      "loss": 0.867,
+      "nll_loss": 0.8503815531730652,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.5240992044922789,
+      "grad_norm": 12.375,
+      "learning_rate": 2.724447119343572e-07,
+      "log_odds_chosen": 0.14831513166427612,
+      "log_odds_ratio": -0.6715076565742493,
+      "logits/chosen": -2.2129743099212646,
+      "logits/rejected": -2.202531337738037,
+      "logps/chosen": -0.3877745270729065,
+      "logps/rejected": -0.4536987841129303,
+      "loss": 0.8995,
+      "nll_loss": 0.9369557499885559,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 560
+    },
+    {
+      "epoch": 0.5334581188582125,
+      "grad_norm": 10.8125,
+      "learning_rate": 2.642944555373965e-07,
+      "log_odds_chosen": 0.15156146883964539,
+      "log_odds_ratio": -0.6531722545623779,
+      "logits/chosen": -2.2276740074157715,
+      "logits/rejected": -2.2312440872192383,
+      "logps/chosen": -0.3907632827758789,
+      "logps/rejected": -0.44457387924194336,
+      "loss": 0.8901,
+      "nll_loss": 0.9654369354248047,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 570
+    },
+    {
+      "epoch": 0.542817033224146,
+      "grad_norm": 12.375,
+      "learning_rate": 2.561289241161095e-07,
+      "log_odds_chosen": 0.16657111048698425,
+      "log_odds_ratio": -0.6478754281997681,
+      "logits/chosen": -2.2165768146514893,
+      "logits/rejected": -2.218228340148926,
+      "logps/chosen": -0.38118991255760193,
+      "logps/rejected": -0.43258100748062134,
+      "loss": 0.9585,
+      "nll_loss": 0.9578523635864258,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 580
+    },
+    {
+      "epoch": 0.5521759475900796,
+      "grad_norm": 10.75,
+      "learning_rate": 2.479568433397441e-07,
+      "log_odds_chosen": 0.13426819443702698,
+      "log_odds_ratio": -0.6695290207862854,
+      "logits/chosen": -2.215862989425659,
+      "logits/rejected": -2.2101454734802246,
+      "logps/chosen": -0.3773101568222046,
+      "logps/rejected": -0.4307495653629303,
+      "loss": 0.8512,
+      "nll_loss": 0.8539910316467285,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 590
+    },
+    {
+      "epoch": 0.5615348619560131,
+      "grad_norm": 12.3125,
+      "learning_rate": 2.3978694587617473e-07,
+      "log_odds_chosen": 0.15946467220783234,
+      "log_odds_ratio": -0.6578431129455566,
+      "logits/chosen": -2.1991055011749268,
+      "logits/rejected": -2.2187142372131348,
+      "logps/chosen": -0.3854084610939026,
+      "logps/rejected": -0.43376803398132324,
+      "loss": 0.8769,
+      "nll_loss": 0.8557795286178589,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.5708937763219466,
+      "grad_norm": 11.4375,
+      "learning_rate": 2.3162796206019266e-07,
+      "log_odds_chosen": 0.19673123955726624,
+      "log_odds_ratio": -0.6394155621528625,
+      "logits/chosen": -2.1414363384246826,
+      "logits/rejected": -2.137732982635498,
+      "logps/chosen": -0.3869827091693878,
+      "logps/rejected": -0.4562526345252991,
+      "loss": 0.8573,
+      "nll_loss": 0.8035401105880737,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 610
+    },
+    {
+      "epoch": 0.5802526906878802,
+      "grad_norm": 13.4375,
+      "learning_rate": 2.2348861056428868e-07,
+      "log_odds_chosen": 0.17950603365898132,
+      "log_odds_ratio": -0.6448289752006531,
+      "logits/chosen": -2.2693517208099365,
+      "logits/rejected": -2.2487759590148926,
+      "logps/chosen": -0.3937278389930725,
+      "logps/rejected": -0.47679153084754944,
+      "loss": 0.8618,
+      "nll_loss": 0.8664249181747437,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 620
+    },
+    {
+      "epoch": 0.5896116050538137,
+      "grad_norm": 12.625,
+      "learning_rate": 2.153775890818989e-07,
+      "log_odds_chosen": 0.16706261038780212,
+      "log_odds_ratio": -0.6368611454963684,
+      "logits/chosen": -2.2624447345733643,
+      "logits/rejected": -2.2497055530548096,
+      "logps/chosen": -0.38325315713882446,
+      "logps/rejected": -0.43356624245643616,
+      "loss": 0.8918,
+      "nll_loss": 0.9218624234199524,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 630
+    },
+    {
+      "epoch": 0.5989705194197473,
+      "grad_norm": 11.4375,
+      "learning_rate": 2.0730356503306806e-07,
+      "log_odds_chosen": 0.06589867919683456,
+      "log_odds_ratio": -0.691096305847168,
+      "logits/chosen": -2.210688352584839,
+      "logits/rejected": -2.207019329071045,
+      "logps/chosen": -0.3885454535484314,
+      "logps/rejected": -0.4102134704589844,
+      "loss": 0.877,
+      "nll_loss": 0.8589447736740112,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 640
+    },
+    {
+      "epoch": 0.6083294337856808,
+      "grad_norm": 12.75,
+      "learning_rate": 1.9927516630246335e-07,
+      "log_odds_chosen": 0.1333846151828766,
+      "log_odds_ratio": -0.6582024693489075,
+      "logits/chosen": -2.276191473007202,
+      "logits/rejected": -2.2583796977996826,
+      "logps/chosen": -0.38965895771980286,
+      "logps/rejected": -0.43281763792037964,
+      "loss": 0.9212,
+      "nll_loss": 0.8427213430404663,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 650
+    },
+    {
+      "epoch": 0.6176883481516144,
+      "grad_norm": 9.4375,
+      "learning_rate": 1.9130097201963545e-07,
+      "log_odds_chosen": 0.2855406403541565,
+      "log_odds_ratio": -0.6136248707771301,
+      "logits/chosen": -2.1809091567993164,
+      "logits/rejected": -2.1479828357696533,
+      "logps/chosen": -0.3822997212409973,
+      "logps/rejected": -0.5103154182434082,
+      "loss": 0.8373,
+      "nll_loss": 0.8126581907272339,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 660
+    },
+    {
+      "epoch": 0.6270472625175479,
+      "grad_norm": 10.9375,
+      "learning_rate": 1.833895033913789e-07,
+      "log_odds_chosen": 0.2555854916572571,
+      "log_odds_ratio": -0.6214571595191956,
+      "logits/chosen": -2.1881003379821777,
+      "logits/rejected": -2.2056005001068115,
+      "logps/chosen": -0.38820981979370117,
+      "logps/rejected": -0.49839335680007935,
+      "loss": 0.882,
+      "nll_loss": 0.8913403749465942,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 670
+    },
+    {
+      "epoch": 0.6364061768834816,
+      "grad_norm": 12.1875,
+      "learning_rate": 1.755492145959896e-07,
+      "log_odds_chosen": 0.27418556809425354,
+      "log_odds_ratio": -0.6061091423034668,
+      "logits/chosen": -2.237029552459717,
+      "logits/rejected": -2.258998155593872,
+      "logps/chosen": -0.36980870366096497,
+      "logps/rejected": -0.47223013639450073,
+      "loss": 0.897,
+      "nll_loss": 0.9085220098495483,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 680
+    },
+    {
+      "epoch": 0.6457650912494151,
+      "grad_norm": 11.5625,
+      "learning_rate": 1.6778848374914728e-07,
+      "log_odds_chosen": 0.1495472937822342,
+      "log_odds_ratio": -0.666257917881012,
+      "logits/chosen": -2.232522964477539,
+      "logits/rejected": -2.2000768184661865,
+      "logps/chosen": -0.3772524297237396,
+      "logps/rejected": -0.43047913908958435,
+      "loss": 0.8478,
+      "nll_loss": 0.8113440275192261,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 690
+    },
+    {
+      "epoch": 0.6551240056153487,
+      "grad_norm": 10.875,
+      "learning_rate": 1.6011560395107998e-07,
+      "log_odds_chosen": 0.1376083940267563,
+      "log_odds_ratio": -0.6573753356933594,
+      "logits/chosen": -2.1723008155822754,
+      "logits/rejected": -2.152540683746338,
+      "logps/chosen": -0.39665332436561584,
+      "logps/rejected": -0.4487191140651703,
+      "loss": 0.8878,
+      "nll_loss": 0.8974568247795105,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.6644829199812822,
+      "grad_norm": 11.5,
+      "learning_rate": 1.5253877442457446e-07,
+      "log_odds_chosen": 0.14467862248420715,
+      "log_odds_ratio": -0.6563876271247864,
+      "logits/chosen": -2.2255406379699707,
+      "logits/rejected": -2.2133126258850098,
+      "logps/chosen": -0.3928825259208679,
+      "logps/rejected": -0.441458523273468,
+      "loss": 0.8912,
+      "nll_loss": 0.8892416954040527,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 710
+    },
+    {
+      "epoch": 0.6738418343472157,
+      "grad_norm": 11.4375,
+      "learning_rate": 1.450660917533048e-07,
+      "log_odds_chosen": 0.16002854704856873,
+      "log_odds_ratio": -0.6535741090774536,
+      "logits/chosen": -2.152003049850464,
+      "logits/rejected": -2.1638240814208984,
+      "logps/chosen": -0.3956072926521301,
+      "logps/rejected": -0.4611222743988037,
+      "loss": 0.9267,
+      "nll_loss": 0.9579558372497559,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 720
+    },
+    {
+      "epoch": 0.6832007487131493,
+      "grad_norm": 11.1875,
+      "learning_rate": 1.377055412298402e-07,
+      "log_odds_chosen": 0.19179068505764008,
+      "log_odds_ratio": -0.6297602653503418,
+      "logits/chosen": -2.2455074787139893,
+      "logits/rejected": -2.249809741973877,
+      "logps/chosen": -0.37641215324401855,
+      "logps/rejected": -0.44604501128196716,
+      "loss": 0.8778,
+      "nll_loss": 0.8855358958244324,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 730
+    },
+    {
+      "epoch": 0.6925596630790828,
+      "grad_norm": 10.8125,
+      "learning_rate": 1.3046498832257924e-07,
+      "log_odds_chosen": 0.30156663060188293,
+      "log_odds_ratio": -0.5952414870262146,
+      "logits/chosen": -2.2041194438934326,
+      "logits/rejected": -2.2143595218658447,
+      "logps/chosen": -0.37330108880996704,
+      "logps/rejected": -0.4847453236579895,
+      "loss": 0.8899,
+      "nll_loss": 0.8489750623703003,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 740
+    },
+    {
+      "epoch": 0.7019185774450164,
+      "grad_norm": 12.9375,
+      "learning_rate": 1.233521702707264e-07,
+      "log_odds_chosen": 0.12726518511772156,
+      "log_odds_ratio": -0.6660614013671875,
+      "logits/chosen": -2.2514965534210205,
+      "logits/rejected": -2.2471282482147217,
+      "logps/chosen": -0.3983449637889862,
+      "logps/rejected": -0.4496460556983948,
+      "loss": 0.8822,
+      "nll_loss": 0.9095920324325562,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 750
+    },
+    {
+      "epoch": 0.7112774918109499,
+      "grad_norm": 9.3125,
+      "learning_rate": 1.1637468781629567e-07,
+      "log_odds_chosen": 0.221353679895401,
+      "log_odds_ratio": -0.6361473798751831,
+      "logits/chosen": -2.201655864715576,
+      "logits/rejected": -2.2004904747009277,
+      "logps/chosen": -0.37590572237968445,
+      "logps/rejected": -0.46796751022338867,
+      "loss": 0.8861,
+      "nll_loss": 0.8721257448196411,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 760
+    },
+    {
+      "epoch": 0.7206364061768835,
+      "grad_norm": 10.375,
+      "learning_rate": 1.0953999708197404e-07,
+      "log_odds_chosen": 0.14084991812705994,
+      "log_odds_ratio": -0.6663475632667542,
+      "logits/chosen": -2.2368383407592773,
+      "logits/rejected": -2.2399377822875977,
+      "logps/chosen": -0.39043423533439636,
+      "logps/rejected": -0.4488556981086731,
+      "loss": 0.8736,
+      "nll_loss": 0.8931602239608765,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 770
+    },
+    {
+      "epoch": 0.729995320542817,
+      "grad_norm": 10.25,
+      "learning_rate": 1.0285540160352404e-07,
+      "log_odds_chosen": 0.10203123092651367,
+      "log_odds_ratio": -0.6751465797424316,
+      "logits/chosen": -2.278337001800537,
+      "logits/rejected": -2.264094591140747,
+      "logps/chosen": -0.3964601159095764,
+      "logps/rejected": -0.4286923408508301,
+      "loss": 0.8769,
+      "nll_loss": 0.9037150144577026,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 780
+    },
+    {
+      "epoch": 0.7393542349087506,
+      "grad_norm": 10.3125,
+      "learning_rate": 9.632804452524256e-08,
+      "log_odds_chosen": 0.17006394267082214,
+      "log_odds_ratio": -0.647351861000061,
+      "logits/chosen": -2.162752628326416,
+      "logits/rejected": -2.163599729537964,
+      "logps/chosen": -0.39570730924606323,
+      "logps/rejected": -0.45942431688308716,
+      "loss": 0.8123,
+      "nll_loss": 0.8693070411682129,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 790
+    },
+    {
+      "epoch": 0.7487131492746841,
+      "grad_norm": 10.4375,
+      "learning_rate": 8.996490096681109e-08,
+      "log_odds_chosen": 0.13158640265464783,
+      "log_odds_ratio": -0.67692631483078,
+      "logits/chosen": -2.180257797241211,
+      "logits/rejected": -2.1624608039855957,
+      "logps/chosen": -0.40238919854164124,
+      "logps/rejected": -0.46038302779197693,
+      "loss": 0.8765,
+      "nll_loss": 0.9388486742973328,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.7580720636406177,
+      "grad_norm": 11.0,
+      "learning_rate": 8.377277056969842e-08,
+      "log_odds_chosen": 0.20461300015449524,
+      "log_odds_ratio": -0.6381627321243286,
+      "logits/chosen": -2.2441630363464355,
+      "logits/rejected": -2.2336671352386475,
+      "logps/chosen": -0.39944490790367126,
+      "logps/rejected": -0.47810035943984985,
+      "loss": 0.9009,
+      "nll_loss": 0.8954687118530273,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 810
+    },
+    {
+      "epoch": 0.7674309780065512,
+      "grad_norm": 12.3125,
+      "learning_rate": 7.775827023107834e-08,
+      "log_odds_chosen": 0.21859991550445557,
+      "log_odds_ratio": -0.6190574765205383,
+      "logits/chosen": -2.2584617137908936,
+      "logits/rejected": -2.235044479370117,
+      "logps/chosen": -0.3853353261947632,
+      "logps/rejected": -0.46209225058555603,
+      "loss": 0.8857,
+      "nll_loss": 0.9639716148376465,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 820
+    },
+    {
+      "epoch": 0.7767898923724847,
+      "grad_norm": 13.75,
+      "learning_rate": 7.192782703302785e-08,
+      "log_odds_chosen": 0.23809826374053955,
+      "log_odds_ratio": -0.6136423349380493,
+      "logits/chosen": -2.222693920135498,
+      "logits/rejected": -2.198784828186035,
+      "logps/chosen": -0.3764819800853729,
+      "logps/rejected": -0.45437926054000854,
+      "loss": 0.8463,
+      "nll_loss": 0.8469756841659546,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 830
+    },
+    {
+      "epoch": 0.7861488067384184,
+      "grad_norm": 9.875,
+      "learning_rate": 6.628767137456067e-08,
+      "log_odds_chosen": 0.1315256655216217,
+      "log_odds_ratio": -0.6661390066146851,
+      "logits/chosen": -2.1875691413879395,
+      "logits/rejected": -2.193073034286499,
+      "logps/chosen": -0.41907310485839844,
+      "logps/rejected": -0.4673064649105072,
+      "loss": 0.9208,
+      "nll_loss": 0.9243777990341187,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 840
+    },
+    {
+      "epoch": 0.7955077211043519,
+      "grad_norm": 13.125,
+      "learning_rate": 6.08438303138365e-08,
+      "log_odds_chosen": 0.0879138857126236,
+      "log_odds_ratio": -0.6789141893386841,
+      "logits/chosen": -2.191913366317749,
+      "logits/rejected": -2.1954355239868164,
+      "logps/chosen": -0.38346320390701294,
+      "logps/rejected": -0.415657103061676,
+      "loss": 0.8978,
+      "nll_loss": 0.9076964259147644,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 850
+    },
+    {
+      "epoch": 0.8048666354702855,
+      "grad_norm": 10.0625,
+      "learning_rate": 5.560212112766011e-08,
+      "log_odds_chosen": 0.14250405132770538,
+      "log_odds_ratio": -0.6655804514884949,
+      "logits/chosen": -2.218059539794922,
+      "logits/rejected": -2.209329843521118,
+      "logps/chosen": -0.386410653591156,
+      "logps/rejected": -0.4412326216697693,
+      "loss": 0.8719,
+      "nll_loss": 0.8908575177192688,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 860
+    },
+    {
+      "epoch": 0.814225549836219,
+      "grad_norm": 11.125,
+      "learning_rate": 5.056814509515092e-08,
+      "log_odds_chosen": 0.20792749524116516,
+      "log_odds_ratio": -0.6451287865638733,
+      "logits/chosen": -2.1958823204040527,
+      "logits/rejected": -2.227415084838867,
+      "logps/chosen": -0.3777526021003723,
+      "logps/rejected": -0.4553712010383606,
+      "loss": 0.8686,
+      "nll_loss": 0.8219059705734253,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 870
+    },
+    {
+      "epoch": 0.8235844642021526,
+      "grad_norm": 11.5625,
+      "learning_rate": 4.57472815122294e-08,
+      "log_odds_chosen": 0.12616851925849915,
+      "log_odds_ratio": -0.6744897961616516,
+      "logits/chosen": -2.239567279815674,
+      "logits/rejected": -2.2245991230010986,
+      "logps/chosen": -0.39220941066741943,
+      "logps/rejected": -0.43573325872421265,
+      "loss": 0.8534,
+      "nll_loss": 0.8611897230148315,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 880
+    },
+    {
+      "epoch": 0.8329433785680861,
+      "grad_norm": 11.0,
+      "learning_rate": 4.1144681943312135e-08,
+      "log_odds_chosen": 0.18745501339435577,
+      "log_odds_ratio": -0.6411550045013428,
+      "logits/chosen": -2.1551897525787354,
+      "logits/rejected": -2.1661386489868164,
+      "logps/chosen": -0.3798125982284546,
+      "logps/rejected": -0.4548855721950531,
+      "loss": 0.9022,
+      "nll_loss": 0.9224790334701538,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 890
+    },
+    {
+      "epoch": 0.8423022929340197,
+      "grad_norm": 10.875,
+      "learning_rate": 3.676526471636168e-08,
+      "log_odds_chosen": 0.20975276827812195,
+      "log_odds_ratio": -0.6299919486045837,
+      "logits/chosen": -2.2081241607666016,
+      "logits/rejected": -2.2052078247070312,
+      "logps/chosen": -0.3888050615787506,
+      "logps/rejected": -0.47621068358421326,
+      "loss": 0.8317,
+      "nll_loss": 0.7904391884803772,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.8516612072999532,
+      "grad_norm": 11.0,
+      "learning_rate": 3.2613709667171893e-08,
+      "log_odds_chosen": 0.1670573353767395,
+      "log_odds_ratio": -0.650361180305481,
+      "logits/chosen": -2.1842589378356934,
+      "logits/rejected": -2.193791151046753,
+      "logps/chosen": -0.38958585262298584,
+      "logps/rejected": -0.44640764594078064,
+      "loss": 0.8918,
+      "nll_loss": 0.8657588958740234,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 910
+    },
+    {
+      "epoch": 0.8610201216658867,
+      "grad_norm": 13.1875,
+      "learning_rate": 2.8694453138505296e-08,
+      "log_odds_chosen": 0.18690498173236847,
+      "log_odds_ratio": -0.643862247467041,
+      "logits/chosen": -2.155745267868042,
+      "logits/rejected": -2.159320116043091,
+      "logps/chosen": -0.379416286945343,
+      "logps/rejected": -0.44661951065063477,
+      "loss": 0.8589,
+      "nll_loss": 0.8909047245979309,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 920
+    },
+    {
+      "epoch": 0.8703790360318203,
+      "grad_norm": 11.625,
+      "learning_rate": 2.5011683239426847e-08,
+      "log_odds_chosen": 0.23082701861858368,
+      "log_odds_ratio": -0.6160436868667603,
+      "logits/chosen": -2.1533544063568115,
+      "logits/rejected": -2.12406587600708,
+      "logps/chosen": -0.3944535255432129,
+      "logps/rejected": -0.47682076692581177,
+      "loss": 0.9318,
+      "nll_loss": 0.9148885607719421,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 930
+    },
+    {
+      "epoch": 0.8797379503977538,
+      "grad_norm": 11.4375,
+      "learning_rate": 2.1569335369899884e-08,
+      "log_odds_chosen": 0.21969418227672577,
+      "log_odds_ratio": -0.6254672408103943,
+      "logits/chosen": -2.2102274894714355,
+      "logits/rejected": -2.197098970413208,
+      "logps/chosen": -0.3689967691898346,
+      "logps/rejected": -0.4529111385345459,
+      "loss": 0.9042,
+      "nll_loss": 0.9422861933708191,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 940
+    },
+    {
+      "epoch": 0.8890968647636874,
+      "grad_norm": 10.3125,
+      "learning_rate": 1.837108801542589e-08,
+      "log_odds_chosen": 0.26736536622047424,
+      "log_odds_ratio": -0.6209851503372192,
+      "logits/chosen": -2.193885564804077,
+      "logits/rejected": -2.1692655086517334,
+      "logps/chosen": -0.3677137494087219,
+      "logps/rejected": -0.49029093980789185,
+      "loss": 0.903,
+      "nll_loss": 0.9059442281723022,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 950
+    },
+    {
+      "epoch": 0.8984557791296209,
+      "grad_norm": 11.75,
+      "learning_rate": 1.5420358816223e-08,
+      "log_odds_chosen": 0.21792039275169373,
+      "log_odds_ratio": -0.6334658861160278,
+      "logits/chosen": -2.1850409507751465,
+      "logits/rejected": -2.1796321868896484,
+      "logps/chosen": -0.3942349851131439,
+      "logps/rejected": -0.4783645272254944,
+      "loss": 0.8978,
+      "nll_loss": 0.8998034596443176,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 960
+    },
+    {
+      "epoch": 0.9078146934955545,
+      "grad_norm": 10.1875,
+      "learning_rate": 1.2720300915142978e-08,
+      "log_odds_chosen": 0.19287154078483582,
+      "log_odds_ratio": -0.632893443107605,
+      "logits/chosen": -2.238175630569458,
+      "logits/rejected": -2.2145469188690186,
+      "logps/chosen": -0.3847096562385559,
+      "logps/rejected": -0.44970273971557617,
+      "loss": 0.945,
+      "nll_loss": 0.8885831832885742,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 970
+    },
+    {
+      "epoch": 0.917173607861488,
+      "grad_norm": 9.5625,
+      "learning_rate": 1.0273799588229659e-08,
+      "log_odds_chosen": 0.14185890555381775,
+      "log_odds_ratio": -0.654270350933075,
+      "logits/chosen": -2.204613208770752,
+      "logits/rejected": -2.192824602127075,
+      "logps/chosen": -0.3829004764556885,
+      "logps/rejected": -0.43003320693969727,
+      "loss": 0.969,
+      "nll_loss": 0.9601060152053833,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 980
+    },
+    {
+      "epoch": 0.9265325222274217,
+      "grad_norm": 10.25,
+      "learning_rate": 8.08346916151903e-09,
+      "log_odds_chosen": 0.18503454327583313,
+      "log_odds_ratio": -0.6544171571731567,
+      "logits/chosen": -2.224045515060425,
+      "logits/rejected": -2.1920361518859863,
+      "logps/chosen": -0.39199355244636536,
+      "logps/rejected": -0.47227340936660767,
+      "loss": 0.8559,
+      "nll_loss": 0.8463509678840637,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 990
+    },
+    {
+      "epoch": 0.9358914365933552,
+      "grad_norm": 11.0,
+      "learning_rate": 6.151650217376347e-09,
+      "log_odds_chosen": 0.2020350694656372,
+      "log_odds_ratio": -0.6279042959213257,
+      "logits/chosen": -2.1893532276153564,
+      "logits/rejected": -2.1703667640686035,
+      "logps/chosen": -0.39548614621162415,
+      "logps/rejected": -0.47544950246810913,
+      "loss": 0.8985,
+      "nll_loss": 0.9111002683639526,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.9452503509592888,
+      "grad_norm": 10.0625,
+      "learning_rate": 4.480407093354566e-09,
+      "log_odds_chosen": 0.12327748537063599,
+      "log_odds_ratio": -0.6677930951118469,
+      "logits/chosen": -2.215139865875244,
+      "logits/rejected": -2.2001559734344482,
+      "logps/chosen": -0.39772313833236694,
+      "logps/rejected": -0.4363015294075012,
+      "loss": 0.9252,
+      "nll_loss": 0.8623770475387573,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1010
+    },
+    {
+      "epoch": 0.9546092653252223,
+      "grad_norm": 8.8125,
+      "learning_rate": 3.0715256762478825e-09,
+      "log_odds_chosen": 0.24486811459064484,
+      "log_odds_ratio": -0.6162024140357971,
+      "logits/chosen": -2.1808042526245117,
+      "logits/rejected": -2.1617355346679688,
+      "logps/chosen": -0.3899694085121155,
+      "logps/rejected": -0.4756743907928467,
+      "loss": 0.887,
+      "nll_loss": 0.8923400640487671,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1020
+    },
+    {
+      "epoch": 0.9639681796911558,
+      "grad_norm": 11.9375,
+      "learning_rate": 1.926511493696936e-09,
+      "log_odds_chosen": 0.11399779468774796,
+      "log_odds_ratio": -0.6707941889762878,
+      "logits/chosen": -2.2159488201141357,
+      "logits/rejected": -2.221107244491577,
+      "logps/chosen": -0.39392104744911194,
+      "logps/rejected": -0.4452263414859772,
+      "loss": 0.9244,
+      "nll_loss": 0.8677811622619629,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1030
+    },
+    {
+      "epoch": 0.9733270940570894,
+      "grad_norm": 11.625,
+      "learning_rate": 1.04658810538516e-09,
+      "log_odds_chosen": 0.18175111711025238,
+      "log_odds_ratio": -0.634798526763916,
+      "logits/chosen": -2.1980602741241455,
+      "logits/rejected": -2.200080394744873,
+      "logps/chosen": -0.36757367849349976,
+      "logps/rejected": -0.42325907945632935,
+      "loss": 0.8552,
+      "nll_loss": 0.8295245170593262,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1040
+    },
+    {
+      "epoch": 0.9826860084230229,
+      "grad_norm": 10.625,
+      "learning_rate": 4.3269579554558455e-10,
+      "log_odds_chosen": 0.10918863862752914,
+      "log_odds_ratio": -0.6753650903701782,
+      "logits/chosen": -2.2247250080108643,
+      "logits/rejected": -2.2069859504699707,
+      "logps/chosen": -0.4014809727668762,
+      "logps/rejected": -0.4421558380126953,
+      "loss": 0.9251,
+      "nll_loss": 0.9922356605529785,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1050
+    },
+    {
+      "epoch": 0.9920449227889565,
+      "grad_norm": 10.625,
+      "learning_rate": 8.549056817513944e-11,
+      "log_odds_chosen": 0.2163115292787552,
+      "log_odds_ratio": -0.6318882703781128,
+      "logits/chosen": -2.2252681255340576,
+      "logits/rejected": -2.2008399963378906,
+      "logps/chosen": -0.39117246866226196,
+      "logps/rejected": -0.4764987528324127,
+      "loss": 0.893,
+      "nll_loss": 0.927023708820343,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1060
+    },
+    {
+      "epoch": 0.9995320542817033,
+      "step": 1068,
+      "total_flos": 0.0,
+      "train_loss": 0.9322090081954271,
+      "train_runtime": 16459.2443,
+      "train_samples_per_second": 2.077,
+      "train_steps_per_second": 0.065
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1068,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}