Model save

README.md

@@ -0,0 +1,59 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: anydef-orpo-v2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# anydef-orpo-v2
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 64
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.43.3
+- Pytorch 2.1.2+cu121
+- Datasets 2.18.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 0.0,
+    "train_loss": 0.16268803322004982,
+    "train_runtime": 7294.4356,
+    "train_samples": 19245,
+    "train_samples_per_second": 7.915,
+    "train_steps_per_second": 0.124
+}

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.43.3"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c722f3309f8baa50abc2ce1353973ac34246fb78946d3bba22313639b2799b8
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:810a00e16834e2c924831fe991d8d995826f401f0e8a27138be4551ec26ca349
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78bd4b5861d69f2ffc1646024509e603fe53ba6f01603f0d3015f5b7fa6f119
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Aug20_08-37-28_hgx1/events.out.tfevents.1724136396.hgx1.154369.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d7cee71c80e7a5a96510e02ff8fee5d41ae0c0880b6f89b7295abb5e5820a83
+size 82866

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 0.0,
+    "train_loss": 0.16268803322004982,
+    "train_runtime": 7294.4356,
+    "train_samples": 19245,
+    "train_samples_per_second": 7.915,
+    "train_steps_per_second": 0.124
+}

trainer_state.json

@@ -0,0 +1,1662 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 903,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03322259136212625,
+      "grad_norm": 516.0,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.16222620010375977,
+      "log_odds_ratio": -0.6306548118591309,
+      "logits/chosen": -2.278585433959961,
+      "logits/rejected": -2.279832124710083,
+      "logps/chosen": -2.096900463104248,
+      "logps/rejected": -2.239978551864624,
+      "loss": 4.4301,
+      "nll_loss": 4.511023998260498,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.10484502464532852,
+      "rewards/margins": 0.007153891958296299,
+      "rewards/rejected": -0.11199891567230225,
+      "step": 10
+    },
+    {
+      "epoch": 0.0664451827242525,
+      "grad_norm": 51.25,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.2909570336341858,
+      "log_odds_ratio": -0.5764315724372864,
+      "logits/chosen": -2.9006943702697754,
+      "logits/rejected": -2.899392604827881,
+      "logps/chosen": -1.8797166347503662,
+      "logps/rejected": -2.132899045944214,
+      "loss": 2.0537,
+      "nll_loss": 1.9548499584197998,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.09398583322763443,
+      "rewards/margins": 0.012659117579460144,
+      "rewards/rejected": -0.10664495080709457,
+      "step": 20
+    },
+    {
+      "epoch": 0.09966777408637874,
+      "grad_norm": 21.5,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.4692462384700775,
+      "log_odds_ratio": -0.510283350944519,
+      "logits/chosen": -2.651613235473633,
+      "logits/rejected": -2.651846170425415,
+      "logps/chosen": -1.1686553955078125,
+      "logps/rejected": -1.4823462963104248,
+      "loss": 0.8489,
+      "nll_loss": 0.7852751612663269,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.058432769030332565,
+      "rewards/margins": 0.015684548765420914,
+      "rewards/rejected": -0.07411732524633408,
+      "step": 30
+    },
+    {
+      "epoch": 0.132890365448505,
+      "grad_norm": 5.4375,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 1.7947794198989868,
+      "log_odds_ratio": -0.26858600974082947,
+      "logits/chosen": -2.2309794425964355,
+      "logits/rejected": -2.229917526245117,
+      "logps/chosen": -0.3180321156978607,
+      "logps/rejected": -1.0096584558486938,
+      "loss": 0.682,
+      "nll_loss": 0.6571449041366577,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.015901606529951096,
+      "rewards/margins": 0.03458131104707718,
+      "rewards/rejected": -0.05048292130231857,
+      "step": 40
+    },
+    {
+      "epoch": 0.16611295681063123,
+      "grad_norm": 3.921875,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 3.033090114593506,
+      "log_odds_ratio": -0.14658799767494202,
+      "logits/chosen": -2.118901014328003,
+      "logits/rejected": -2.1173033714294434,
+      "logps/chosen": -0.25798267126083374,
+      "logps/rejected": -1.627856969833374,
+      "loss": 0.6248,
+      "nll_loss": 0.600904107093811,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.012899133376777172,
+      "rewards/margins": 0.06849371641874313,
+      "rewards/rejected": -0.08139285445213318,
+      "step": 50
+    },
+    {
+      "epoch": 0.19933554817275748,
+      "grad_norm": 6.03125,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 3.421691417694092,
+      "log_odds_ratio": -0.14014041423797607,
+      "logits/chosen": -2.077253580093384,
+      "logits/rejected": -2.0778613090515137,
+      "logps/chosen": -0.22530755400657654,
+      "logps/rejected": -1.8401321172714233,
+      "loss": 0.6023,
+      "nll_loss": 0.6070769429206848,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.011265376582741737,
+      "rewards/margins": 0.08074121922254562,
+      "rewards/rejected": -0.0920066088438034,
+      "step": 60
+    },
+    {
+      "epoch": 0.23255813953488372,
+      "grad_norm": 4.40625,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 4.030662536621094,
+      "log_odds_ratio": -0.07589299231767654,
+      "logits/chosen": -2.0758180618286133,
+      "logits/rejected": -2.0744235515594482,
+      "logps/chosen": -0.2279697209596634,
+      "logps/rejected": -2.004603385925293,
+      "loss": 0.5389,
+      "nll_loss": 0.534622073173523,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.011398485861718655,
+      "rewards/margins": 0.08883167803287506,
+      "rewards/rejected": -0.10023017227649689,
+      "step": 70
+    },
+    {
+      "epoch": 0.26578073089701,
+      "grad_norm": 3.609375,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 4.552683353424072,
+      "log_odds_ratio": -0.056131958961486816,
+      "logits/chosen": -1.9744676351547241,
+      "logits/rejected": -1.9743177890777588,
+      "logps/chosen": -0.18155953288078308,
+      "logps/rejected": -2.2489826679229736,
+      "loss": 0.5203,
+      "nll_loss": 0.4936625063419342,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.009077977389097214,
+      "rewards/margins": 0.10337115824222565,
+      "rewards/rejected": -0.11244914680719376,
+      "step": 80
+    },
+    {
+      "epoch": 0.29900332225913623,
+      "grad_norm": 4.09375,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 4.036518096923828,
+      "log_odds_ratio": -0.11566118150949478,
+      "logits/chosen": -2.0107204914093018,
+      "logits/rejected": -2.009970188140869,
+      "logps/chosen": -0.2278076857328415,
+      "logps/rejected": -2.3444762229919434,
+      "loss": 0.4353,
+      "nll_loss": 0.42819660902023315,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.01139038521796465,
+      "rewards/margins": 0.10583342611789703,
+      "rewards/rejected": -0.11722382158041,
+      "step": 90
+    },
+    {
+      "epoch": 0.33222591362126247,
+      "grad_norm": 5.59375,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 5.044631004333496,
+      "log_odds_ratio": -0.04541964456439018,
+      "logits/chosen": -1.9699828624725342,
+      "logits/rejected": -1.973362922668457,
+      "logps/chosen": -0.17292837798595428,
+      "logps/rejected": -2.476628065109253,
+      "loss": 0.378,
+      "nll_loss": 0.4115411639213562,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008646419271826744,
+      "rewards/margins": 0.11518500000238419,
+      "rewards/rejected": -0.12383142858743668,
+      "step": 100
+    },
+    {
+      "epoch": 0.3654485049833887,
+      "grad_norm": 5.15625,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 5.11702823638916,
+      "log_odds_ratio": -0.06080981343984604,
+      "logits/chosen": -1.883387565612793,
+      "logits/rejected": -1.8867937326431274,
+      "logps/chosen": -0.2399381399154663,
+      "logps/rejected": -2.785667896270752,
+      "loss": 0.3143,
+      "nll_loss": 0.25671663880348206,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.011996905319392681,
+      "rewards/margins": 0.12728647887706757,
+      "rewards/rejected": -0.13928338885307312,
+      "step": 110
+    },
+    {
+      "epoch": 0.39867109634551495,
+      "grad_norm": 5.0625,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 5.193495273590088,
+      "log_odds_ratio": -0.06189825385808945,
+      "logits/chosen": -2.042405605316162,
+      "logits/rejected": -2.0465760231018066,
+      "logps/chosen": -0.1781654804944992,
+      "logps/rejected": -2.7364754676818848,
+      "loss": 0.2883,
+      "nll_loss": 0.31883668899536133,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.008908274583518505,
+      "rewards/margins": 0.12791548669338226,
+      "rewards/rejected": -0.13682377338409424,
+      "step": 120
+    },
+    {
+      "epoch": 0.4318936877076412,
+      "grad_norm": 6.40625,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 5.065199375152588,
+      "log_odds_ratio": -0.08915611356496811,
+      "logits/chosen": -1.965550184249878,
+      "logits/rejected": -1.9655630588531494,
+      "logps/chosen": -0.2024417221546173,
+      "logps/rejected": -2.5124077796936035,
+      "loss": 0.2346,
+      "nll_loss": 0.2726798355579376,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.01012208592146635,
+      "rewards/margins": 0.11549830436706543,
+      "rewards/rejected": -0.12562039494514465,
+      "step": 130
+    },
+    {
+      "epoch": 0.46511627906976744,
+      "grad_norm": 5.65625,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 6.224053859710693,
+      "log_odds_ratio": -0.03768063336610794,
+      "logits/chosen": -1.9541261196136475,
+      "logits/rejected": -1.9546234607696533,
+      "logps/chosen": -0.18753428757190704,
+      "logps/rejected": -3.1887362003326416,
+      "loss": 0.2075,
+      "nll_loss": 0.1721208095550537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.009376714006066322,
+      "rewards/margins": 0.15006008744239807,
+      "rewards/rejected": -0.15943679213523865,
+      "step": 140
+    },
+    {
+      "epoch": 0.4983388704318937,
+      "grad_norm": 5.53125,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 5.947408199310303,
+      "log_odds_ratio": -0.03485158830881119,
+      "logits/chosen": -1.928086280822754,
+      "logits/rejected": -1.9305970668792725,
+      "logps/chosen": -0.16427160799503326,
+      "logps/rejected": -3.250744581222534,
+      "loss": 0.1935,
+      "nll_loss": 0.1781485378742218,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008213580586016178,
+      "rewards/margins": 0.15432362258434296,
+      "rewards/rejected": -0.16253721714019775,
+      "step": 150
+    },
+    {
+      "epoch": 0.53156146179402,
+      "grad_norm": 5.0625,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 5.442956924438477,
+      "log_odds_ratio": -0.0749388113617897,
+      "logits/chosen": -2.025440216064453,
+      "logits/rejected": -2.0270285606384277,
+      "logps/chosen": -0.18180342018604279,
+      "logps/rejected": -2.965075731277466,
+      "loss": 0.1607,
+      "nll_loss": 0.18244585394859314,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.00909017026424408,
+      "rewards/margins": 0.13916362822055817,
+      "rewards/rejected": -0.14825379848480225,
+      "step": 160
+    },
+    {
+      "epoch": 0.5647840531561462,
+      "grad_norm": 4.8125,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 6.501151084899902,
+      "log_odds_ratio": -0.04770870879292488,
+      "logits/chosen": -1.9193477630615234,
+      "logits/rejected": -1.9223600625991821,
+      "logps/chosen": -0.16726627945899963,
+      "logps/rejected": -3.4573769569396973,
+      "loss": 0.1339,
+      "nll_loss": 0.16945432126522064,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.008363312110304832,
+      "rewards/margins": 0.16450552642345428,
+      "rewards/rejected": -0.17286884784698486,
+      "step": 170
+    },
+    {
+      "epoch": 0.5980066445182725,
+      "grad_norm": 4.8125,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 6.667401313781738,
+      "log_odds_ratio": -0.049022819846868515,
+      "logits/chosen": -1.849381685256958,
+      "logits/rejected": -1.8515217304229736,
+      "logps/chosen": -0.135690376162529,
+      "logps/rejected": -3.4136643409729004,
+      "loss": 0.12,
+      "nll_loss": 0.0964335948228836,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.006784519646316767,
+      "rewards/margins": 0.16389869153499603,
+      "rewards/rejected": -0.17068320512771606,
+      "step": 180
+    },
+    {
+      "epoch": 0.6312292358803987,
+      "grad_norm": 4.0625,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 6.884723663330078,
+      "log_odds_ratio": -0.028747648000717163,
+      "logits/chosen": -1.9872407913208008,
+      "logits/rejected": -1.9903675317764282,
+      "logps/chosen": -0.1553266942501068,
+      "logps/rejected": -3.7033779621124268,
+      "loss": 0.1064,
+      "nll_loss": 0.1674821376800537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007766333874315023,
+      "rewards/margins": 0.1774025708436966,
+      "rewards/rejected": -0.18516890704631805,
+      "step": 190
+    },
+    {
+      "epoch": 0.6644518272425249,
+      "grad_norm": 4.40625,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 6.214459419250488,
+      "log_odds_ratio": -0.04790915921330452,
+      "logits/chosen": -1.8185112476348877,
+      "logits/rejected": -1.820067048072815,
+      "logps/chosen": -0.15120986104011536,
+      "logps/rejected": -3.4781315326690674,
+      "loss": 0.0925,
+      "nll_loss": 0.06244741007685661,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.007560492493212223,
+      "rewards/margins": 0.16634607315063477,
+      "rewards/rejected": -0.17390656471252441,
+      "step": 200
+    },
+    {
+      "epoch": 0.6976744186046512,
+      "grad_norm": 5.46875,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 7.0073442459106445,
+      "log_odds_ratio": -0.021652357652783394,
+      "logits/chosen": -1.8007183074951172,
+      "logits/rejected": -1.8030471801757812,
+      "logps/chosen": -0.15487684309482574,
+      "logps/rejected": -3.859619140625,
+      "loss": 0.0897,
+      "nll_loss": 0.07238463312387466,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007743841968476772,
+      "rewards/margins": 0.1852371096611023,
+      "rewards/rejected": -0.19298096001148224,
+      "step": 210
+    },
+    {
+      "epoch": 0.7308970099667774,
+      "grad_norm": 3.484375,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 6.155528545379639,
+      "log_odds_ratio": -0.03304092958569527,
+      "logits/chosen": -1.827543020248413,
+      "logits/rejected": -1.8281749486923218,
+      "logps/chosen": -0.13896045088768005,
+      "logps/rejected": -3.173088312149048,
+      "loss": 0.0762,
+      "nll_loss": 0.08422436565160751,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0069480217061936855,
+      "rewards/margins": 0.15170639753341675,
+      "rewards/rejected": -0.15865442156791687,
+      "step": 220
+    },
+    {
+      "epoch": 0.7641196013289037,
+      "grad_norm": 5.03125,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 7.247349739074707,
+      "log_odds_ratio": -0.028882017359137535,
+      "logits/chosen": -1.8549703359603882,
+      "logits/rejected": -1.854103446006775,
+      "logps/chosen": -0.13769736886024475,
+      "logps/rejected": -3.765294313430786,
+      "loss": 0.0634,
+      "nll_loss": 0.058101166039705276,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.006884869188070297,
+      "rewards/margins": 0.18137982487678528,
+      "rewards/rejected": -0.18826469779014587,
+      "step": 230
+    },
+    {
+      "epoch": 0.7973421926910299,
+      "grad_norm": 3.875,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 7.331563472747803,
+      "log_odds_ratio": -0.05921437218785286,
+      "logits/chosen": -1.9100215435028076,
+      "logits/rejected": -1.9127085208892822,
+      "logps/chosen": -0.12960004806518555,
+      "logps/rejected": -3.6168124675750732,
+      "loss": 0.0571,
+      "nll_loss": 0.05991581082344055,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.00648000231012702,
+      "rewards/margins": 0.17436063289642334,
+      "rewards/rejected": -0.1808406263589859,
+      "step": 240
+    },
+    {
+      "epoch": 0.8305647840531561,
+      "grad_norm": 3.53125,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 7.729872703552246,
+      "log_odds_ratio": -0.01583888754248619,
+      "logits/chosen": -1.9329345226287842,
+      "logits/rejected": -1.9311659336090088,
+      "logps/chosen": -0.16400082409381866,
+      "logps/rejected": -4.614955902099609,
+      "loss": 0.0623,
+      "nll_loss": 0.045454978942871094,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008200041949748993,
+      "rewards/margins": 0.2225477695465088,
+      "rewards/rejected": -0.230747789144516,
+      "step": 250
+    },
+    {
+      "epoch": 0.8637873754152824,
+      "grad_norm": 3.15625,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 7.562008857727051,
+      "log_odds_ratio": -0.06357506662607193,
+      "logits/chosen": -1.8379156589508057,
+      "logits/rejected": -1.8368641138076782,
+      "logps/chosen": -0.15229454636573792,
+      "logps/rejected": -4.107020854949951,
+      "loss": 0.0485,
+      "nll_loss": 0.033093374222517014,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0076147266663610935,
+      "rewards/margins": 0.19773633778095245,
+      "rewards/rejected": -0.2053510695695877,
+      "step": 260
+    },
+    {
+      "epoch": 0.8970099667774086,
+      "grad_norm": 3.15625,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 7.212728023529053,
+      "log_odds_ratio": -0.07752545177936554,
+      "logits/chosen": -1.7939443588256836,
+      "logits/rejected": -1.7951726913452148,
+      "logps/chosen": -0.16109412908554077,
+      "logps/rejected": -3.8484885692596436,
+      "loss": 0.0557,
+      "nll_loss": 0.05208224803209305,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.008054706268012524,
+      "rewards/margins": 0.18436971306800842,
+      "rewards/rejected": -0.19242441654205322,
+      "step": 270
+    },
+    {
+      "epoch": 0.9302325581395349,
+      "grad_norm": 9.25,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 7.317690372467041,
+      "log_odds_ratio": -0.03297095373272896,
+      "logits/chosen": -1.7949488162994385,
+      "logits/rejected": -1.7990652322769165,
+      "logps/chosen": -0.13610824942588806,
+      "logps/rejected": -3.9614219665527344,
+      "loss": 0.0511,
+      "nll_loss": 0.05410151928663254,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0068054115399718285,
+      "rewards/margins": 0.19126567244529724,
+      "rewards/rejected": -0.19807109236717224,
+      "step": 280
+    },
+    {
+      "epoch": 0.9634551495016611,
+      "grad_norm": 3.796875,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 7.124808311462402,
+      "log_odds_ratio": -0.059395015239715576,
+      "logits/chosen": -1.7850925922393799,
+      "logits/rejected": -1.7865415811538696,
+      "logps/chosen": -0.12057201564311981,
+      "logps/rejected": -3.576596736907959,
+      "loss": 0.0428,
+      "nll_loss": 0.03806814178824425,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0060286009684205055,
+      "rewards/margins": 0.17280122637748718,
+      "rewards/rejected": -0.1788298487663269,
+      "step": 290
+    },
+    {
+      "epoch": 0.9966777408637874,
+      "grad_norm": 3.65625,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 8.637829780578613,
+      "log_odds_ratio": -0.03425002470612526,
+      "logits/chosen": -1.8364009857177734,
+      "logits/rejected": -1.8421306610107422,
+      "logps/chosen": -0.09547251462936401,
+      "logps/rejected": -4.495790481567383,
+      "loss": 0.0425,
+      "nll_loss": 0.04902663081884384,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.004773625638335943,
+      "rewards/margins": 0.22001588344573975,
+      "rewards/rejected": -0.22478953003883362,
+      "step": 300
+    },
+    {
+      "epoch": 1.0299003322259137,
+      "grad_norm": 2.8125,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 7.561132907867432,
+      "log_odds_ratio": -0.03642101213335991,
+      "logits/chosen": -1.7316315174102783,
+      "logits/rejected": -1.7338483333587646,
+      "logps/chosen": -0.0959225445985794,
+      "logps/rejected": -4.059569358825684,
+      "loss": 0.0292,
+      "nll_loss": 0.028634298592805862,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0047961268573999405,
+      "rewards/margins": 0.1981823742389679,
+      "rewards/rejected": -0.2029784917831421,
+      "step": 310
+    },
+    {
+      "epoch": 1.06312292358804,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 9.18364429473877,
+      "log_odds_ratio": -0.014110135845839977,
+      "logits/chosen": -1.7149658203125,
+      "logits/rejected": -1.7151539325714111,
+      "logps/chosen": -0.12401266396045685,
+      "logps/rejected": -5.025017738342285,
+      "loss": 0.0279,
+      "nll_loss": 0.019647331908345222,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0062006330117583275,
+      "rewards/margins": 0.24505026638507843,
+      "rewards/rejected": -0.25125089287757874,
+      "step": 320
+    },
+    {
+      "epoch": 1.0963455149501662,
+      "grad_norm": 3.78125,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 8.793200492858887,
+      "log_odds_ratio": -0.009492707438766956,
+      "logits/chosen": -1.7698380947113037,
+      "logits/rejected": -1.770939588546753,
+      "logps/chosen": -0.11295183002948761,
+      "logps/rejected": -4.82761812210083,
+      "loss": 0.0332,
+      "nll_loss": 0.02587791346013546,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.005647591315209866,
+      "rewards/margins": 0.23573331534862518,
+      "rewards/rejected": -0.24138090014457703,
+      "step": 330
+    },
+    {
+      "epoch": 1.1295681063122924,
+      "grad_norm": 1.78125,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 8.388921737670898,
+      "log_odds_ratio": -0.009843870997428894,
+      "logits/chosen": -1.7839330434799194,
+      "logits/rejected": -1.7847379446029663,
+      "logps/chosen": -0.0786074846982956,
+      "logps/rejected": -4.0679216384887695,
+      "loss": 0.0277,
+      "nll_loss": 0.022286545485258102,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003930374514311552,
+      "rewards/margins": 0.19946573674678802,
+      "rewards/rejected": -0.20339611172676086,
+      "step": 340
+    },
+    {
+      "epoch": 1.1627906976744187,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 8.371394157409668,
+      "log_odds_ratio": -0.016807865351438522,
+      "logits/chosen": -1.829673171043396,
+      "logits/rejected": -1.8303911685943604,
+      "logps/chosen": -0.10171355307102203,
+      "logps/rejected": -4.102365016937256,
+      "loss": 0.0254,
+      "nll_loss": 0.019215276464819908,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.005085677839815617,
+      "rewards/margins": 0.20003259181976318,
+      "rewards/rejected": -0.20511826872825623,
+      "step": 350
+    },
+    {
+      "epoch": 1.196013289036545,
+      "grad_norm": 3.484375,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 8.600504875183105,
+      "log_odds_ratio": -0.01216288935393095,
+      "logits/chosen": -1.7985435724258423,
+      "logits/rejected": -1.8070056438446045,
+      "logps/chosen": -0.06962008774280548,
+      "logps/rejected": -4.320959568023682,
+      "loss": 0.0267,
+      "nll_loss": 0.029016951099038124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0034810048528015614,
+      "rewards/margins": 0.21256697177886963,
+      "rewards/rejected": -0.2160479724407196,
+      "step": 360
+    },
+    {
+      "epoch": 1.2292358803986712,
+      "grad_norm": 3.1875,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 9.181499481201172,
+      "log_odds_ratio": -0.009369775652885437,
+      "logits/chosen": -1.7924983501434326,
+      "logits/rejected": -1.7953475713729858,
+      "logps/chosen": -0.11782409995794296,
+      "logps/rejected": -4.833601474761963,
+      "loss": 0.0282,
+      "nll_loss": 0.02526969090104103,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0058912052772939205,
+      "rewards/margins": 0.23578886687755585,
+      "rewards/rejected": -0.2416801005601883,
+      "step": 370
+    },
+    {
+      "epoch": 1.2624584717607974,
+      "grad_norm": 2.796875,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 9.619462966918945,
+      "log_odds_ratio": -0.039833612740039825,
+      "logits/chosen": -1.8319047689437866,
+      "logits/rejected": -1.8300600051879883,
+      "logps/chosen": -0.10069389641284943,
+      "logps/rejected": -5.014215469360352,
+      "loss": 0.0266,
+      "nll_loss": 0.028005924075841904,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.005034694913774729,
+      "rewards/margins": 0.24567607045173645,
+      "rewards/rejected": -0.25071078538894653,
+      "step": 380
+    },
+    {
+      "epoch": 1.2956810631229236,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 8.534521102905273,
+      "log_odds_ratio": -0.01909024640917778,
+      "logits/chosen": -1.8544292449951172,
+      "logits/rejected": -1.8542238473892212,
+      "logps/chosen": -0.08696131408214569,
+      "logps/rejected": -4.430135250091553,
+      "loss": 0.0232,
+      "nll_loss": 0.02147216536104679,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.004348065238445997,
+      "rewards/margins": 0.217158704996109,
+      "rewards/rejected": -0.2215067595243454,
+      "step": 390
+    },
+    {
+      "epoch": 1.3289036544850499,
+      "grad_norm": 1.640625,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 8.600648880004883,
+      "log_odds_ratio": -0.012287040241062641,
+      "logits/chosen": -1.8202216625213623,
+      "logits/rejected": -1.8218141794204712,
+      "logps/chosen": -0.09084287285804749,
+      "logps/rejected": -4.572846412658691,
+      "loss": 0.0247,
+      "nll_loss": 0.02059631608426571,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004542144015431404,
+      "rewards/margins": 0.22410018742084503,
+      "rewards/rejected": -0.22864234447479248,
+      "step": 400
+    },
+    {
+      "epoch": 1.3621262458471761,
+      "grad_norm": 3.640625,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 9.651135444641113,
+      "log_odds_ratio": -0.00363975390791893,
+      "logits/chosen": -1.7791026830673218,
+      "logits/rejected": -1.7837848663330078,
+      "logps/chosen": -0.06081225723028183,
+      "logps/rejected": -4.943568706512451,
+      "loss": 0.024,
+      "nll_loss": 0.021141935139894485,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0030406129080802202,
+      "rewards/margins": 0.24413780868053436,
+      "rewards/rejected": -0.24717843532562256,
+      "step": 410
+    },
+    {
+      "epoch": 1.3953488372093024,
+      "grad_norm": 1.875,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 8.751879692077637,
+      "log_odds_ratio": -0.010004991665482521,
+      "logits/chosen": -1.8006718158721924,
+      "logits/rejected": -1.804359793663025,
+      "logps/chosen": -0.0964532420039177,
+      "logps/rejected": -4.5417680740356445,
+      "loss": 0.0242,
+      "nll_loss": 0.025043126195669174,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004822662565857172,
+      "rewards/margins": 0.22226576507091522,
+      "rewards/rejected": -0.22708842158317566,
+      "step": 420
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 1.984375,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 9.289121627807617,
+      "log_odds_ratio": -0.015525879338383675,
+      "logits/chosen": -1.7658954858779907,
+      "logits/rejected": -1.7667725086212158,
+      "logps/chosen": -0.08038794249296188,
+      "logps/rejected": -4.9636030197143555,
+      "loss": 0.0201,
+      "nll_loss": 0.020869722589850426,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004019397310912609,
+      "rewards/margins": 0.24416080117225647,
+      "rewards/rejected": -0.24818019568920135,
+      "step": 430
+    },
+    {
+      "epoch": 1.4617940199335548,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 8.703948974609375,
+      "log_odds_ratio": -0.03381601721048355,
+      "logits/chosen": -1.8417913913726807,
+      "logits/rejected": -1.845391869544983,
+      "logps/chosen": -0.09374421089887619,
+      "logps/rejected": -4.612320899963379,
+      "loss": 0.0229,
+      "nll_loss": 0.021974634379148483,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.004687210079282522,
+      "rewards/margins": 0.22592882812023163,
+      "rewards/rejected": -0.23061604797840118,
+      "step": 440
+    },
+    {
+      "epoch": 1.495016611295681,
+      "grad_norm": 1.890625,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 9.317599296569824,
+      "log_odds_ratio": -0.008442175574600697,
+      "logits/chosen": -1.9028446674346924,
+      "logits/rejected": -1.9078031778335571,
+      "logps/chosen": -0.0840989276766777,
+      "logps/rejected": -4.985965251922607,
+      "loss": 0.0277,
+      "nll_loss": 0.019282350316643715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0042049465700984,
+      "rewards/margins": 0.24509334564208984,
+      "rewards/rejected": -0.24929828941822052,
+      "step": 450
+    },
+    {
+      "epoch": 1.5282392026578073,
+      "grad_norm": 1.109375,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 8.135089874267578,
+      "log_odds_ratio": -0.014614465646445751,
+      "logits/chosen": -1.80266535282135,
+      "logits/rejected": -1.8127800226211548,
+      "logps/chosen": -0.08102138340473175,
+      "logps/rejected": -4.199796676635742,
+      "loss": 0.0208,
+      "nll_loss": 0.02294105850160122,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004051069263368845,
+      "rewards/margins": 0.20593877136707306,
+      "rewards/rejected": -0.20998983085155487,
+      "step": 460
+    },
+    {
+      "epoch": 1.5614617940199336,
+      "grad_norm": 2.40625,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 9.386737823486328,
+      "log_odds_ratio": -0.008776369504630566,
+      "logits/chosen": -1.7513538599014282,
+      "logits/rejected": -1.7554657459259033,
+      "logps/chosen": -0.06230410188436508,
+      "logps/rejected": -4.882990837097168,
+      "loss": 0.0227,
+      "nll_loss": 0.01421122532337904,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003115205094218254,
+      "rewards/margins": 0.24103431403636932,
+      "rewards/rejected": -0.24414952099323273,
+      "step": 470
+    },
+    {
+      "epoch": 1.5946843853820598,
+      "grad_norm": 1.625,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 8.983893394470215,
+      "log_odds_ratio": -0.024142052978277206,
+      "logits/chosen": -1.744749665260315,
+      "logits/rejected": -1.7481634616851807,
+      "logps/chosen": -0.06949851661920547,
+      "logps/rejected": -4.313258171081543,
+      "loss": 0.0222,
+      "nll_loss": 0.014674236066639423,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0034749258775264025,
+      "rewards/margins": 0.2121879756450653,
+      "rewards/rejected": -0.2156629115343094,
+      "step": 480
+    },
+    {
+      "epoch": 1.627906976744186,
+      "grad_norm": 2.0,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 9.338783264160156,
+      "log_odds_ratio": -0.003996879793703556,
+      "logits/chosen": -1.791486382484436,
+      "logits/rejected": -1.795069694519043,
+      "logps/chosen": -0.06457408517599106,
+      "logps/rejected": -4.7583208084106445,
+      "loss": 0.0177,
+      "nll_loss": 0.015971561893820763,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0032287046778947115,
+      "rewards/margins": 0.23468737304210663,
+      "rewards/rejected": -0.23791606724262238,
+      "step": 490
+    },
+    {
+      "epoch": 1.6611295681063123,
+      "grad_norm": 1.6015625,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 9.166845321655273,
+      "log_odds_ratio": -0.019674357026815414,
+      "logits/chosen": -1.7548977136611938,
+      "logits/rejected": -1.7554121017456055,
+      "logps/chosen": -0.10164159536361694,
+      "logps/rejected": -4.969311714172363,
+      "loss": 0.0212,
+      "nll_loss": 0.01509636640548706,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.00508207967504859,
+      "rewards/margins": 0.243383526802063,
+      "rewards/rejected": -0.24846558272838593,
+      "step": 500
+    },
+    {
+      "epoch": 1.6943521594684385,
+      "grad_norm": 1.28125,
+      "learning_rate": 2.2140372138502386e-06,
+      "log_odds_chosen": 8.379947662353516,
+      "log_odds_ratio": -0.03198238089680672,
+      "logits/chosen": -1.8583186864852905,
+      "logits/rejected": -1.8610032796859741,
+      "logps/chosen": -0.08590197563171387,
+      "logps/rejected": -4.655932426452637,
+      "loss": 0.0195,
+      "nll_loss": 0.017975686118006706,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0042950985953211784,
+      "rewards/margins": 0.22850151360034943,
+      "rewards/rejected": -0.23279662430286407,
+      "step": 510
+    },
+    {
+      "epoch": 1.7275747508305648,
+      "grad_norm": 3.125,
+      "learning_rate": 2.1926450482675734e-06,
+      "log_odds_chosen": 8.935117721557617,
+      "log_odds_ratio": -0.014949078671634197,
+      "logits/chosen": -1.7289230823516846,
+      "logits/rejected": -1.73250412940979,
+      "logps/chosen": -0.07164986431598663,
+      "logps/rejected": -4.507022857666016,
+      "loss": 0.0178,
+      "nll_loss": 0.014084184542298317,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003582493169233203,
+      "rewards/margins": 0.22176864743232727,
+      "rewards/rejected": -0.22535113990306854,
+      "step": 520
+    },
+    {
+      "epoch": 1.760797342192691,
+      "grad_norm": 1.71875,
+      "learning_rate": 2.1718612138153473e-06,
+      "log_odds_chosen": 10.034872055053711,
+      "log_odds_ratio": -0.006512313149869442,
+      "logits/chosen": -1.7052526473999023,
+      "logits/rejected": -1.7084852457046509,
+      "logps/chosen": -0.07591713964939117,
+      "logps/rejected": -5.4831976890563965,
+      "loss": 0.0223,
+      "nll_loss": 0.020342020317912102,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003795857075601816,
+      "rewards/margins": 0.27036404609680176,
+      "rewards/rejected": -0.27415987849235535,
+      "step": 530
+    },
+    {
+      "epoch": 1.7940199335548173,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.151657414559676e-06,
+      "log_odds_chosen": 8.771195411682129,
+      "log_odds_ratio": -0.020360399037599564,
+      "logits/chosen": -1.7692371606826782,
+      "logits/rejected": -1.772956132888794,
+      "logps/chosen": -0.07401047646999359,
+      "logps/rejected": -4.339105129241943,
+      "loss": 0.0207,
+      "nll_loss": 0.02204059436917305,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0037005238700658083,
+      "rewards/margins": 0.21325473487377167,
+      "rewards/rejected": -0.2169552594423294,
+      "step": 540
+    },
+    {
+      "epoch": 1.8272425249169435,
+      "grad_norm": 2.4375,
+      "learning_rate": 2.132007163556104e-06,
+      "log_odds_chosen": 8.97265625,
+      "log_odds_ratio": -0.010379938408732414,
+      "logits/chosen": -1.761279821395874,
+      "logits/rejected": -1.7625354528427124,
+      "logps/chosen": -0.08797116577625275,
+      "logps/rejected": -4.506954669952393,
+      "loss": 0.0191,
+      "nll_loss": 0.015043037012219429,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004398558288812637,
+      "rewards/margins": 0.2209491729736328,
+      "rewards/rejected": -0.22534772753715515,
+      "step": 550
+    },
+    {
+      "epoch": 1.8604651162790697,
+      "grad_norm": 1.96875,
+      "learning_rate": 2.1128856368212917e-06,
+      "log_odds_chosen": 9.888033866882324,
+      "log_odds_ratio": -0.00335489958524704,
+      "logits/chosen": -1.7607628107070923,
+      "logits/rejected": -1.7624956369400024,
+      "logps/chosen": -0.06476293504238129,
+      "logps/rejected": -5.111817359924316,
+      "loss": 0.0194,
+      "nll_loss": 0.013030583038926125,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003238147124648094,
+      "rewards/margins": 0.2523527443408966,
+      "rewards/rejected": -0.25559088587760925,
+      "step": 560
+    },
+    {
+      "epoch": 1.893687707641196,
+      "grad_norm": 1.6328125,
+      "learning_rate": 2.0942695414584777e-06,
+      "log_odds_chosen": 8.265462875366211,
+      "log_odds_ratio": -0.017327692359685898,
+      "logits/chosen": -1.7851202487945557,
+      "logits/rejected": -1.7864612340927124,
+      "logps/chosen": -0.10817401111125946,
+      "logps/rejected": -4.388330459594727,
+      "loss": 0.0191,
+      "nll_loss": 0.01878109760582447,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.005408700555562973,
+      "rewards/margins": 0.21400780975818634,
+      "rewards/rejected": -0.2194165289402008,
+      "step": 570
+    },
+    {
+      "epoch": 1.9269102990033222,
+      "grad_norm": 1.2265625,
+      "learning_rate": 2.0761369963434992e-06,
+      "log_odds_chosen": 8.885993003845215,
+      "log_odds_ratio": -0.027743179351091385,
+      "logits/chosen": -1.7333558797836304,
+      "logits/rejected": -1.7339531183242798,
+      "logps/chosen": -0.1336405724287033,
+      "logps/rejected": -4.482719421386719,
+      "loss": 0.0201,
+      "nll_loss": 0.015746701508760452,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0066820294596254826,
+      "rewards/margins": 0.21745392680168152,
+      "rewards/rejected": -0.22413596510887146,
+      "step": 580
+    },
+    {
+      "epoch": 1.9601328903654485,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.058467423981546e-06,
+      "log_odds_chosen": 9.54238224029541,
+      "log_odds_ratio": -0.018449265509843826,
+      "logits/chosen": -1.7866191864013672,
+      "logits/rejected": -1.7889735698699951,
+      "logps/chosen": -0.10259035974740982,
+      "logps/rejected": -4.987481117248535,
+      "loss": 0.0187,
+      "nll_loss": 0.0172494538128376,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.005129518453031778,
+      "rewards/margins": 0.2442445456981659,
+      "rewards/rejected": -0.24937407672405243,
+      "step": 590
+    },
+    {
+      "epoch": 1.9933554817275747,
+      "grad_norm": 0.890625,
+      "learning_rate": 2.0412414523193154e-06,
+      "log_odds_chosen": 9.205097198486328,
+      "log_odds_ratio": -0.005630264058709145,
+      "logits/chosen": -1.7373039722442627,
+      "logits/rejected": -1.7389856576919556,
+      "logps/chosen": -0.09646005928516388,
+      "logps/rejected": -4.7054619789123535,
+      "loss": 0.0182,
+      "nll_loss": 0.02860497497022152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004823002498596907,
+      "rewards/margins": 0.2304501086473465,
+      "rewards/rejected": -0.235273078083992,
+      "step": 600
+    },
+    {
+      "epoch": 2.026578073089701,
+      "grad_norm": 1.4921875,
+      "learning_rate": 2.0244408254472904e-06,
+      "log_odds_chosen": 10.055734634399414,
+      "log_odds_ratio": -0.011299138888716698,
+      "logits/chosen": -1.7703691720962524,
+      "logits/rejected": -1.771695852279663,
+      "logps/chosen": -0.06641928851604462,
+      "logps/rejected": -5.32825231552124,
+      "loss": 0.017,
+      "nll_loss": 0.013218941166996956,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0033209645189344883,
+      "rewards/margins": 0.26309165358543396,
+      "rewards/rejected": -0.2664126455783844,
+      "step": 610
+    },
+    {
+      "epoch": 2.0598006644518274,
+      "grad_norm": 1.3671875,
+      "learning_rate": 2.0080483222562476e-06,
+      "log_odds_chosen": 10.432465553283691,
+      "log_odds_ratio": -0.016602743417024612,
+      "logits/chosen": -1.8167240619659424,
+      "logits/rejected": -1.8178882598876953,
+      "logps/chosen": -0.05541493743658066,
+      "logps/rejected": -5.0302910804748535,
+      "loss": 0.0145,
+      "nll_loss": 0.015641603618860245,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0027707472909241915,
+      "rewards/margins": 0.24874380230903625,
+      "rewards/rejected": -0.25151461362838745,
+      "step": 620
+    },
+    {
+      "epoch": 2.0930232558139537,
+      "grad_norm": 0.7421875,
+      "learning_rate": 1.9920476822239895e-06,
+      "log_odds_chosen": 10.32500171661377,
+      "log_odds_ratio": -0.0108437929302454,
+      "logits/chosen": -1.6814196109771729,
+      "logits/rejected": -1.6834462881088257,
+      "logps/chosen": -0.07114370167255402,
+      "logps/rejected": -5.47824764251709,
+      "loss": 0.0146,
+      "nll_loss": 0.012577347457408905,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003557185409590602,
+      "rewards/margins": 0.270355224609375,
+      "rewards/rejected": -0.27391237020492554,
+      "step": 630
+    },
+    {
+      "epoch": 2.12624584717608,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.976423537605237e-06,
+      "log_odds_chosen": 9.84516429901123,
+      "log_odds_ratio": -0.007125245872884989,
+      "logits/chosen": -1.823743224143982,
+      "logits/rejected": -1.8290736675262451,
+      "logps/chosen": -0.06525563448667526,
+      "logps/rejected": -5.315881729125977,
+      "loss": 0.0157,
+      "nll_loss": 0.014731844887137413,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0032627820037305355,
+      "rewards/margins": 0.2625313103199005,
+      "rewards/rejected": -0.2657940983772278,
+      "step": 640
+    },
+    {
+      "epoch": 2.159468438538206,
+      "grad_norm": 1.140625,
+      "learning_rate": 1.961161351381841e-06,
+      "log_odds_chosen": 11.63329029083252,
+      "log_odds_ratio": -0.0004948956775479019,
+      "logits/chosen": -1.805872917175293,
+      "logits/rejected": -1.8104311227798462,
+      "logps/chosen": -0.04712063446640968,
+      "logps/rejected": -6.125610828399658,
+      "loss": 0.0142,
+      "nll_loss": 0.013872918672859669,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0023560314439237118,
+      "rewards/margins": 0.3039245009422302,
+      "rewards/rejected": -0.30628055334091187,
+      "step": 650
+    },
+    {
+      "epoch": 2.1926910299003324,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.9462473604038077e-06,
+      "log_odds_chosen": 10.331804275512695,
+      "log_odds_ratio": -0.015263216570019722,
+      "logits/chosen": -1.8137140274047852,
+      "logits/rejected": -1.8163013458251953,
+      "logps/chosen": -0.05831971764564514,
+      "logps/rejected": -5.453424453735352,
+      "loss": 0.0144,
+      "nll_loss": 0.014606691896915436,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.002915985882282257,
+      "rewards/margins": 0.26975521445274353,
+      "rewards/rejected": -0.2726712226867676,
+      "step": 660
+    },
+    {
+      "epoch": 2.2259136212624586,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.9316685232156397e-06,
+      "log_odds_chosen": 10.702049255371094,
+      "log_odds_ratio": -0.018240805715322495,
+      "logits/chosen": -1.913975715637207,
+      "logits/rejected": -1.9187599420547485,
+      "logps/chosen": -0.07275418192148209,
+      "logps/rejected": -5.977799892425537,
+      "loss": 0.0142,
+      "nll_loss": 0.015033453702926636,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0036377091892063618,
+      "rewards/margins": 0.2952522933483124,
+      "rewards/rejected": -0.29889002442359924,
+      "step": 670
+    },
+    {
+      "epoch": 2.259136212624585,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.917412472118426e-06,
+      "log_odds_chosen": 11.560079574584961,
+      "log_odds_ratio": -0.0027265329845249653,
+      "logits/chosen": -1.849691390991211,
+      "logits/rejected": -1.8556410074234009,
+      "logps/chosen": -0.05384901165962219,
+      "logps/rejected": -6.353396415710449,
+      "loss": 0.0171,
+      "nll_loss": 0.02180541306734085,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0026924503035843372,
+      "rewards/margins": 0.31497737765312195,
+      "rewards/rejected": -0.3176698088645935,
+      "step": 680
+    },
+    {
+      "epoch": 2.292358803986711,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.9034674690672024e-06,
+      "log_odds_chosen": 11.148561477661133,
+      "log_odds_ratio": -0.00223861588165164,
+      "logits/chosen": -1.8589222431182861,
+      "logits/rejected": -1.8631727695465088,
+      "logps/chosen": -0.06614092737436295,
+      "logps/rejected": -5.854241371154785,
+      "loss": 0.0151,
+      "nll_loss": 0.01388646848499775,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0033070463687181473,
+      "rewards/margins": 0.28940504789352417,
+      "rewards/rejected": -0.2927120625972748,
+      "step": 690
+    },
+    {
+      "epoch": 2.3255813953488373,
+      "grad_norm": 0.72265625,
+      "learning_rate": 1.8898223650461362e-06,
+      "log_odds_chosen": 10.924173355102539,
+      "log_odds_ratio": -0.006120534148067236,
+      "logits/chosen": -1.833099365234375,
+      "logits/rejected": -1.841202974319458,
+      "logps/chosen": -0.045662157237529755,
+      "logps/rejected": -5.663559913635254,
+      "loss": 0.0141,
+      "nll_loss": 0.014723509550094604,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0022831077221781015,
+      "rewards/margins": 0.2808949053287506,
+      "rewards/rejected": -0.2831780016422272,
+      "step": 700
+    },
+    {
+      "epoch": 2.3588039867109636,
+      "grad_norm": 1.546875,
+      "learning_rate": 1.876466562602004e-06,
+      "log_odds_chosen": 11.659400939941406,
+      "log_odds_ratio": -0.007115496788173914,
+      "logits/chosen": -1.8497368097305298,
+      "logits/rejected": -1.8600342273712158,
+      "logps/chosen": -0.043242715299129486,
+      "logps/rejected": -5.790225028991699,
+      "loss": 0.014,
+      "nll_loss": 0.014149373397231102,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002162135671824217,
+      "rewards/margins": 0.28734907507896423,
+      "rewards/rejected": -0.28951120376586914,
+      "step": 710
+    },
+    {
+      "epoch": 2.39202657807309,
+      "grad_norm": 1.703125,
+      "learning_rate": 1.863389981249825e-06,
+      "log_odds_chosen": 11.449972152709961,
+      "log_odds_ratio": -0.005292683839797974,
+      "logits/chosen": -1.9128930568695068,
+      "logits/rejected": -1.9192262887954712,
+      "logps/chosen": -0.043063901364803314,
+      "logps/rejected": -5.712512016296387,
+      "loss": 0.0141,
+      "nll_loss": 0.014921635389328003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0021531949751079082,
+      "rewards/margins": 0.2834724187850952,
+      "rewards/rejected": -0.2856256365776062,
+      "step": 720
+    },
+    {
+      "epoch": 2.425249169435216,
+      "grad_norm": 3.125,
+      "learning_rate": 1.8505830254940132e-06,
+      "log_odds_chosen": 10.498836517333984,
+      "log_odds_ratio": -0.004357654135674238,
+      "logits/chosen": -1.881967306137085,
+      "logits/rejected": -1.8853543996810913,
+      "logps/chosen": -0.03361859172582626,
+      "logps/rejected": -5.354216575622559,
+      "loss": 0.0141,
+      "nll_loss": 0.011369029060006142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0016809297958388925,
+      "rewards/margins": 0.26602986454963684,
+      "rewards/rejected": -0.2677108347415924,
+      "step": 730
+    },
+    {
+      "epoch": 2.4584717607973423,
+      "grad_norm": 1.5625,
+      "learning_rate": 1.8380365552345197e-06,
+      "log_odds_chosen": 10.925847053527832,
+      "log_odds_ratio": -0.003954787738621235,
+      "logits/chosen": -1.8275858163833618,
+      "logits/rejected": -1.8305232524871826,
+      "logps/chosen": -0.06278284639120102,
+      "logps/rejected": -5.769686698913574,
+      "loss": 0.0141,
+      "nll_loss": 0.013375637121498585,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003139142645522952,
+      "rewards/margins": 0.285345196723938,
+      "rewards/rejected": -0.2884843945503235,
+      "step": 740
+    },
+    {
+      "epoch": 2.4916943521594686,
+      "grad_norm": 0.640625,
+      "learning_rate": 1.8257418583505536e-06,
+      "log_odds_chosen": 10.625173568725586,
+      "log_odds_ratio": -0.0035576275549829006,
+      "logits/chosen": -1.7775007486343384,
+      "logits/rejected": -1.7832441329956055,
+      "logps/chosen": -0.04924372583627701,
+      "logps/rejected": -5.4554243087768555,
+      "loss": 0.0151,
+      "nll_loss": 0.015531172044575214,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002462186384946108,
+      "rewards/margins": 0.2703090310096741,
+      "rewards/rejected": -0.2727712094783783,
+      "step": 750
+    },
+    {
+      "epoch": 2.524916943521595,
+      "grad_norm": 0.72265625,
+      "learning_rate": 1.8136906252750293e-06,
+      "log_odds_chosen": 11.038446426391602,
+      "log_odds_ratio": -0.0013516563922166824,
+      "logits/chosen": -1.81307053565979,
+      "logits/rejected": -1.8172311782836914,
+      "logps/chosen": -0.037691373378038406,
+      "logps/rejected": -5.5027947425842285,
+      "loss": 0.0155,
+      "nll_loss": 0.011486930772662163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0018845684826374054,
+      "rewards/margins": 0.2732551693916321,
+      "rewards/rejected": -0.275139719247818,
+      "step": 760
+    },
+    {
+      "epoch": 2.558139534883721,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.801874925391118e-06,
+      "log_odds_chosen": 11.234697341918945,
+      "log_odds_ratio": -0.005816595163196325,
+      "logits/chosen": -1.8077905178070068,
+      "logits/rejected": -1.8116910457611084,
+      "logps/chosen": -0.06007402017712593,
+      "logps/rejected": -5.977658748626709,
+      "loss": 0.0135,
+      "nll_loss": 0.013816078193485737,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0030037013348191977,
+      "rewards/margins": 0.2958792448043823,
+      "rewards/rejected": -0.29888293147087097,
+      "step": 770
+    },
+    {
+      "epoch": 2.5913621262458473,
+      "grad_norm": 0.65234375,
+      "learning_rate": 1.7902871850985824e-06,
+      "log_odds_chosen": 11.535958290100098,
+      "log_odds_ratio": -0.009717768058180809,
+      "logits/chosen": -1.8791577816009521,
+      "logits/rejected": -1.883548378944397,
+      "logps/chosen": -0.051692645996809006,
+      "logps/rejected": -5.989034652709961,
+      "loss": 0.014,
+      "nll_loss": 0.014420375227928162,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0025846322532743216,
+      "rewards/margins": 0.29686713218688965,
+      "rewards/rejected": -0.2994517385959625,
+      "step": 780
+    },
+    {
+      "epoch": 2.6245847176079735,
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.7789201674120502e-06,
+      "log_odds_chosen": 10.751108169555664,
+      "log_odds_ratio": -0.01122227031737566,
+      "logits/chosen": -1.8293044567108154,
+      "logits/rejected": -1.8323638439178467,
+      "logps/chosen": -0.05979070067405701,
+      "logps/rejected": -5.497213363647461,
+      "loss": 0.0157,
+      "nll_loss": 0.014203609898686409,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0029895349871367216,
+      "rewards/margins": 0.2718711495399475,
+      "rewards/rejected": -0.274860680103302,
+      "step": 790
+    },
+    {
+      "epoch": 2.6578073089700998,
+      "grad_norm": 0.5859375,
+      "learning_rate": 1.7677669529663689e-06,
+      "log_odds_chosen": 10.65892219543457,
+      "log_odds_ratio": -0.006628723349422216,
+      "logits/chosen": -1.8738857507705688,
+      "logits/rejected": -1.877375602722168,
+      "logps/chosen": -0.06150083988904953,
+      "logps/rejected": -5.296011924743652,
+      "loss": 0.0149,
+      "nll_loss": 0.013364692218601704,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0030750418081879616,
+      "rewards/margins": 0.2617255747318268,
+      "rewards/rejected": -0.2648006081581116,
+      "step": 800
+    },
+    {
+      "epoch": 2.691029900332226,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.7568209223157664e-06,
+      "log_odds_chosen": 11.236889839172363,
+      "log_odds_ratio": -0.004805346950888634,
+      "logits/chosen": -1.9045976400375366,
+      "logits/rejected": -1.9087021350860596,
+      "logps/chosen": -0.047368817031383514,
+      "logps/rejected": -5.490727424621582,
+      "loss": 0.0147,
+      "nll_loss": 0.0136332456022501,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0023684408515691757,
+      "rewards/margins": 0.2721679210662842,
+      "rewards/rejected": -0.2745364010334015,
+      "step": 810
+    },
+    {
+      "epoch": 2.7242524916943522,
+      "grad_norm": 0.76953125,
+      "learning_rate": 1.7460757394239458e-06,
+      "log_odds_chosen": 11.060879707336426,
+      "log_odds_ratio": -0.0016804604092612863,
+      "logits/chosen": -1.8651930093765259,
+      "logits/rejected": -1.8689038753509521,
+      "logps/chosen": -0.036452341824769974,
+      "logps/rejected": -5.505632400512695,
+      "loss": 0.013,
+      "nll_loss": 0.011724123731255531,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0018226171378046274,
+      "rewards/margins": 0.2734590172767639,
+      "rewards/rejected": -0.2752816081047058,
+      "step": 820
+    },
+    {
+      "epoch": 2.7574750830564785,
+      "grad_norm": 1.3828125,
+      "learning_rate": 1.7355253362515584e-06,
+      "log_odds_chosen": 12.019643783569336,
+      "log_odds_ratio": -0.0025712151546031237,
+      "logits/chosen": -1.9404821395874023,
+      "logits/rejected": -1.9456230401992798,
+      "logps/chosen": -0.04571037366986275,
+      "logps/rejected": -6.3564581871032715,
+      "loss": 0.0144,
+      "nll_loss": 0.01494914572685957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0022855184506624937,
+      "rewards/margins": 0.31553739309310913,
+      "rewards/rejected": -0.3178229033946991,
+      "step": 830
+    },
+    {
+      "epoch": 2.7906976744186047,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.7251638983558855e-06,
+      "log_odds_chosen": 10.963711738586426,
+      "log_odds_ratio": -0.004456724040210247,
+      "logits/chosen": -1.8980438709259033,
+      "logits/rejected": -1.9045253992080688,
+      "logps/chosen": -0.04593021795153618,
+      "logps/rejected": -5.473552703857422,
+      "loss": 0.0145,
+      "nll_loss": 0.017903735861182213,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002296511083841324,
+      "rewards/margins": 0.27138110995292664,
+      "rewards/rejected": -0.27367764711380005,
+      "step": 840
+    },
+    {
+      "epoch": 2.823920265780731,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.7149858514250883e-06,
+      "log_odds_chosen": 10.605644226074219,
+      "log_odds_ratio": -0.004892362747341394,
+      "logits/chosen": -1.8568840026855469,
+      "logits/rejected": -1.867110013961792,
+      "logps/chosen": -0.0643405169248581,
+      "logps/rejected": -5.792882919311523,
+      "loss": 0.0137,
+      "nll_loss": 0.01375966053456068,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0032170258928090334,
+      "rewards/margins": 0.2864271104335785,
+      "rewards/rejected": -0.28964415192604065,
+      "step": 850
+    },
+    {
+      "epoch": 2.857142857142857,
+      "grad_norm": 0.578125,
+      "learning_rate": 1.704985848676184e-06,
+      "log_odds_chosen": 10.766576766967773,
+      "log_odds_ratio": -0.01473341602832079,
+      "logits/chosen": -1.8542922735214233,
+      "logits/rejected": -1.8589084148406982,
+      "logps/chosen": -0.07984187453985214,
+      "logps/rejected": -5.777710914611816,
+      "loss": 0.0152,
+      "nll_loss": 0.020643722265958786,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.00399209326133132,
+      "rewards/margins": 0.2848934531211853,
+      "rewards/rejected": -0.28888556361198425,
+      "step": 860
+    },
+    {
+      "epoch": 2.8903654485049834,
+      "grad_norm": 0.4921875,
+      "learning_rate": 1.6951587590520263e-06,
+      "log_odds_chosen": 11.57789421081543,
+      "log_odds_ratio": -0.0173480324447155,
+      "logits/chosen": -1.7418874502182007,
+      "logits/rejected": -1.7448689937591553,
+      "logps/chosen": -0.054336708039045334,
+      "logps/rejected": -5.907016754150391,
+      "loss": 0.0129,
+      "nll_loss": 0.010191375389695168,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.002716835355386138,
+      "rewards/margins": 0.2926340401172638,
+      "rewards/rejected": -0.2953508496284485,
+      "step": 870
+    },
+    {
+      "epoch": 2.9235880398671097,
+      "grad_norm": 0.6875,
+      "learning_rate": 1.6854996561581053e-06,
+      "log_odds_chosen": 11.569136619567871,
+      "log_odds_ratio": -0.011355452239513397,
+      "logits/chosen": -1.959159255027771,
+      "logits/rejected": -1.9617881774902344,
+      "logps/chosen": -0.06317956745624542,
+      "logps/rejected": -6.309741020202637,
+      "loss": 0.0135,
+      "nll_loss": 0.014484817162156105,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0031589786522090435,
+      "rewards/margins": 0.3123281002044678,
+      "rewards/rejected": -0.3154870867729187,
+      "step": 880
+    },
+    {
+      "epoch": 2.956810631229236,
+      "grad_norm": 0.5625,
+      "learning_rate": 1.6760038078849776e-06,
+      "log_odds_chosen": 11.66430377960205,
+      "log_odds_ratio": -0.001645472482778132,
+      "logits/chosen": -1.8734182119369507,
+      "logits/rejected": -1.8795799016952515,
+      "logps/chosen": -0.04230424761772156,
+      "logps/rejected": -5.725351333618164,
+      "loss": 0.0143,
+      "nll_loss": 0.013855007477104664,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002115212380886078,
+      "rewards/margins": 0.28415238857269287,
+      "rewards/rejected": -0.28626757860183716,
+      "step": 890
+    },
+    {
+      "epoch": 2.990033222591362,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.6666666666666667e-06,
+      "log_odds_chosen": 11.357660293579102,
+      "log_odds_ratio": -0.0027510782238096,
+      "logits/chosen": -1.856702446937561,
+      "logits/rejected": -1.8652187585830688,
+      "logps/chosen": -0.05658254772424698,
+      "logps/rejected": -5.779760360717773,
+      "loss": 0.0168,
+      "nll_loss": 0.03194582462310791,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0028291274793446064,
+      "rewards/margins": 0.2861589193344116,
+      "rewards/rejected": -0.28898805379867554,
+      "step": 900
+    },
+    {
+      "epoch": 3.0,
+      "step": 903,
+      "total_flos": 0.0,
+      "train_loss": 0.16268803322004982,
+      "train_runtime": 7294.4356,
+      "train_samples_per_second": 7.915,
+      "train_steps_per_second": 0.124
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 903,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916af5787c083179b647536627641ffd6ab24535fc3a7ea8f8625f7ac46dc852
+size 5496