Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.4181
+- Loss: 1.4187
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| No log        | 0     | 0    | 1.4181          |
+| No log        | 0     | 0    | 1.4187          |
 
 
 ### Framework versions

adapter_config.json

@@ -16,10 +16,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
     "k_proj",
     "v_proj",
-    "q_proj"
+    "q_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[PAD]": 32000
+}

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 0,
-    "eval_loss": 1.418121576309204,
-    "eval_runtime": 35.9445,
+    "eval_loss": 1.4187188148498535,
+    "eval_runtime": 58.7586,
     "eval_samples": 1200,
-    "eval_samples_per_second": 33.385,
-    "eval_steps_per_second": 4.173,
-    "train_loss": 0.2214554250240326,
-    "train_runtime": 145.6296,
+    "eval_samples_per_second": 20.423,
+    "eval_steps_per_second": 2.553,
+    "train_loss": 0.3546818494796753,
+    "train_runtime": 239.2844,
     "train_samples": 1200,
-    "train_samples_per_second": 8.24,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 5.015,
+    "train_steps_per_second": 0.008
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0,
-    "eval_loss": 1.418121576309204,
-    "eval_runtime": 35.9445,
+    "eval_loss": 1.4187188148498535,
+    "eval_runtime": 58.7586,
     "eval_samples": 1200,
-    "eval_samples_per_second": 33.385,
-    "eval_steps_per_second": 4.173
+    "eval_samples_per_second": 20.423,
+    "eval_steps_per_second": 2.553
 }

runs/Nov30_17-06-41_hpc-hblab/events.out.tfevents.1701338897.hpc-hblab.1611604.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e2c0afb882b5025f67294a97983b1a3091f587d9a71486c1feeb2a820f3b7f8
+size 4806

runs/Nov30_17-06-41_hpc-hblab/events.out.tfevents.1701339195.hpc-hblab.1611604.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3789884d1a74153773224e357c4048d459894ae8378bf7a4eaac07dad685c121
+size 344

special_tokens_map.json

@@ -13,7 +13,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "</s>",
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -29,6 +29,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32000,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -23,6 +25,14 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32000": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [],
@@ -31,8 +41,9 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
-  "model_max_length": 2048,
-  "pad_token": "</s>",
+  "model_max_length": 1024,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0,
-    "train_loss": 0.2214554250240326,
-    "train_runtime": 145.6296,
+    "train_loss": 0.3546818494796753,
+    "train_runtime": 239.2844,
     "train_samples": 1200,
-    "train_samples_per_second": 8.24,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 5.015,
+    "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -10,27 +10,27 @@
   "log_history": [
     {
       "epoch": 0,
-      "eval_loss": 1.4181212186813354,
-      "eval_runtime": 35.8986,
-      "eval_samples_per_second": 33.427,
-      "eval_steps_per_second": 4.178,
+      "eval_loss": 1.418718695640564,
+      "eval_runtime": 59.2437,
+      "eval_samples_per_second": 20.255,
+      "eval_steps_per_second": 2.532,
       "step": 0
     },
     {
       "epoch": 0,
       "step": 0,
-      "total_flos": 6955618468364288.0,
-      "train_loss": 0.2214554250240326,
-      "train_runtime": 145.6296,
-      "train_samples_per_second": 8.24,
-      "train_steps_per_second": 0.014
+      "total_flos": 1.1269869630652416e+16,
+      "train_loss": 0.3546818494796753,
+      "train_runtime": 239.2844,
+      "train_samples_per_second": 5.015,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 5,
   "max_steps": 2,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 6955618468364288.0,
+  "total_flos": 1.1269869630652416e+16,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1852d234f44667a230b4cb1c06479e3c7d7069f8107216ef8fa926ec3a32c806
-size 4664
+oid sha256:c0d11ab835e1773ef1f902c0fcbcfa5a15ea8e274293fd54cc24ad7877072741
+size 4600