Model save

README.md

@@ -1,7 +1,7 @@
 ---
 tags:
 - trl
-- dpo
+- reward-trainer
 - generated_from_trainer
 model-index:
 - name: opt-350m

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.8458248774210612,
-    "train_runtime": 38.2645,
+    "train_loss": 0.8802837530771891,
+    "train_runtime": 29.5366,
     "train_samples": 42,
-    "train_samples_per_second": 1.098,
-    "train_steps_per_second": 0.078
+    "train_samples_per_second": 1.422,
+    "train_steps_per_second": 0.102
 }

config.json

@@ -4,7 +4,7 @@
   "activation_dropout": 0.0,
   "activation_function": "relu",
   "architectures": [
-    "OPTForCausalLM"
+    "OPTForSequenceClassification"
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 2,
@@ -14,7 +14,13 @@
   "eos_token_id": 2,
   "ffn_dim": 4096,
   "hidden_size": 1024,
+  "id2label": {
+    "0": "LABEL_0"
+  },
   "init_std": 0.02,
+  "label2id": {
+    "LABEL_0": 0
+  },
   "layer_norm_elementwise_affine": true,
   "layerdrop": 0.0,
   "max_position_embeddings": 2048,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96757bee906119aac800b9f547e51e7c5ce2ccce2d4097f0dc0f59a533e2b3f6
-size 1324830880
+oid sha256:5c806859e833e81d5b6cdd2855af6e39e3c4ee38e323a169bba6278af096a35c
+size 1324833016

runs/Apr25_11-53-16_MacBook-Pro-3.local/events.out.tfevents.1714038821.MacBook-Pro-3.local.39724.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cba84c12cfbf3327a8119c1970eb529c48384b8c921e428486c478851eba8f4a
+size 4790

runs/Apr25_11-56-04_MacBook-Pro-3.local/events.out.tfevents.1714038969.MacBook-Pro-3.local.42437.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861b865147a18089a72d026b052ad42c9bf1d2a2c3a86af2e2ed99fad9d3961d
+size 4790

runs/Apr25_12-02-11_MacBook-Pro-3.local/events.out.tfevents.1714039360.MacBook-Pro-3.local.48181.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38890d11aba1534b849e52d5c9293ded553dd4ef7d1907ba01e2597829968485
+size 5138

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.8458248774210612,
-    "train_runtime": 38.2645,
+    "train_loss": 0.8802837530771891,
+    "train_runtime": 29.5366,
     "train_samples": 42,
-    "train_samples_per_second": 1.098,
-    "train_steps_per_second": 0.078
+    "train_samples_per_second": 1.422,
+    "train_steps_per_second": 0.102
 }

trainer_state.json

@@ -8,29 +8,14 @@
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.33,
-      "grad_norm": 10.981945037841797,
-      "learning_rate": 5e-07,
-      "logits/chosen": -0.6707152128219604,
-      "logits/rejected": -0.8389236927032471,
-      "logps/chosen": -465.56787109375,
-      "logps/rejected": -1116.6866455078125,
-      "loss": 1.1531,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.2850288152694702,
-      "rewards/margins": 0.3632635474205017,
-      "rewards/rejected": -0.07823469489812851,
-      "step": 1
-    },
     {
       "epoch": 1.0,
       "step": 3,
       "total_flos": 0.0,
-      "train_loss": 0.8458248774210612,
-      "train_runtime": 38.2645,
-      "train_samples_per_second": 1.098,
-      "train_steps_per_second": 0.078
+      "train_loss": 0.8802837530771891,
+      "train_runtime": 29.5366,
+      "train_samples_per_second": 1.422,
+      "train_steps_per_second": 0.102
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3f7f97f31ef1eb375dd9b7308555e225d5387fb7fe51d12f7410ebb4a4eb480
-size 5048
+oid sha256:266c603d9bd4ca09eb2054cb68cd0b1c9bc9c5d6f6cebd43a95fe484aef40d38
+size 4920