Model save

README.md

@@ -1,4 +1,5 @@
 ---
+library_name: transformers
 license: mit
 base_model: FacebookAI/xlm-roberta-large
 tags:
@@ -17,8 +18,8 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [FacebookAI/xlm-roberta-large](https://huggingface.co/FacebookAI/xlm-roberta-large) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6748
-- F1: 0.8262
+- F1: 0.8154
+- Loss: 0.6651
 
 ## Model description
 
@@ -37,29 +38,30 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 2e-05
-- train_batch_size: 32
-- eval_batch_size: 32
+- learning_rate: 1e-05
+- train_batch_size: 64
+- eval_batch_size: 64
 - seed: 42
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 600
 - num_epochs: 5.0
+- mixed_precision_training: Native AMP
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | F1     |
-|:-------------:|:-----:|:----:|:---------------:|:------:|
-| 1.2213        | 1.0   | 1076 | 0.7617          | 0.7825 |
-| 0.6795        | 2.0   | 2152 | 0.6679          | 0.8069 |
-| 0.4878        | 3.0   | 3228 | 0.6416          | 0.8185 |
-| 0.339         | 4.0   | 4304 | 0.6522          | 0.8263 |
-| 0.226         | 5.0   | 5380 | 0.6748          | 0.8262 |
+| Training Loss | Epoch | Step | F1     | Validation Loss |
+|:-------------:|:-----:|:----:|:------:|:---------------:|
+| 1.7444        | 1.0   | 538  | 0.7686 | 0.8148          |
+| 0.7749        | 2.0   | 1076 | 0.8000 | 0.7104          |
+| 0.6165        | 3.0   | 1614 | 0.8114 | 0.6536          |
+| 0.5044        | 4.0   | 2152 | 0.8140 | 0.6571          |
+| 0.4217        | 5.0   | 2690 | 0.8154 | 0.6651          |
 
 
 ### Framework versions
 
-- Transformers 4.43.1
-- Pytorch 2.3.1+cu121
-- Datasets 2.20.0
-- Tokenizers 0.19.1
+- Transformers 4.49.0.dev0
+- Pytorch 2.1.2.post304
+- Datasets 3.2.0
+- Tokenizers 0.21.0

all_results.json

@@ -1,15 +1,15 @@
 {
     "epoch": 5.0,
-    "eval_f1": 0.8262395269824653,
-    "eval_loss": 0.6748126149177551,
-    "eval_runtime": 1120.7242,
+    "eval_f1": 0.8113637968918935,
+    "eval_loss": 0.6536160707473755,
+    "eval_runtime": 21.9128,
     "eval_samples": 8605,
-    "eval_samples_per_second": 7.678,
-    "eval_steps_per_second": 0.24,
+    "eval_samples_per_second": 392.692,
+    "eval_steps_per_second": 6.161,
     "total_flos": 1.603929743474688e+17,
-    "train_loss": 0.5906805073904725,
-    "train_runtime": 93341.3352,
+    "train_loss": 0.7819821552716223,
+    "train_runtime": 1776.2036,
     "train_samples": 34420,
-    "train_samples_per_second": 1.844,
-    "train_steps_per_second": 0.058
+    "train_samples_per_second": 96.892,
+    "train_steps_per_second": 1.514
 }

config.json

@@ -59,7 +59,7 @@
   "position_embedding_type": "absolute",
   "problem_type": "single_label_classification",
   "torch_dtype": "float32",
-  "transformers_version": "4.43.1",
+  "transformers_version": "4.49.0.dev0",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 250002

eval_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 5.0,
-    "eval_f1": 0.8262395269824653,
-    "eval_loss": 0.6748126149177551,
-    "eval_runtime": 1120.7242,
+    "eval_f1": 0.8113637968918935,
+    "eval_loss": 0.6536160707473755,
+    "eval_runtime": 21.9128,
     "eval_samples": 8605,
-    "eval_samples_per_second": 7.678,
-    "eval_steps_per_second": 0.24
+    "eval_samples_per_second": 392.692,
+    "eval_steps_per_second": 6.161
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:387b5a1e432a9111c78c7a8ee75c63b13aafb3fb3ba0562e0e15334df97732c0
+oid sha256:d5cb6f50bd2fb5332a437a955993eaf0bec472eb9b113867fb1081317267532a
 size 2239676072

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ffb37461c391f096759f4a9bbbc329da0f36952f88bab061fcf84940c022e98
-size 17082999
+oid sha256:3a56def25aa40facc030ea8b0b87f3688e4b3c39eb8b45d5702b3a1300fe2a20
+size 17082734

tokenizer_config.json

@@ -42,9 +42,10 @@
     }
   },
   "bos_token": "<s>",
-  "clean_up_tokenization_spaces": true,
+  "clean_up_tokenization_spaces": false,
   "cls_token": "<s>",
   "eos_token": "</s>",
+  "extra_special_tokens": {},
   "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 5.0,
     "total_flos": 1.603929743474688e+17,
-    "train_loss": 0.5906805073904725,
-    "train_runtime": 93341.3352,
+    "train_loss": 0.7819821552716223,
+    "train_runtime": 1776.2036,
     "train_samples": 34420,
-    "train_samples_per_second": 1.844,
-    "train_steps_per_second": 0.058
+    "train_samples_per_second": 96.892,
+    "train_steps_per_second": 1.514
 }

trainer_state.json

@@ -1,105 +1,105 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
+  "best_metric": 0.6536160707473755,
+  "best_model_checkpoint": "./SDG/checkpoint-1614",
   "epoch": 5.0,
-  "eval_steps": 500.0,
-  "global_step": 5380,
+  "eval_steps": 500,
+  "global_step": 2690,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "grad_norm": 17.019458770751953,
-      "learning_rate": 1.8008368200836822e-05,
-      "loss": 1.2213,
-      "step": 1076
+      "epoch": 0.929368029739777,
+      "grad_norm": 26.18483543395996,
+      "learning_rate": 8.266666666666667e-06,
+      "loss": 1.7444,
+      "step": 500
     },
     {
       "epoch": 1.0,
-      "eval_f1": 0.7824958430254769,
-      "eval_loss": 0.7617404460906982,
-      "eval_runtime": 1102.1746,
-      "eval_samples_per_second": 7.807,
-      "eval_steps_per_second": 0.244,
-      "step": 1076
+      "eval_f1": 0.7685770625858339,
+      "eval_loss": 0.8148266077041626,
+      "eval_runtime": 22.0031,
+      "eval_samples_per_second": 391.081,
+      "eval_steps_per_second": 6.136,
+      "step": 538
     },
     {
-      "epoch": 2.0,
-      "grad_norm": 21.141799926757812,
-      "learning_rate": 1.3506276150627616e-05,
-      "loss": 0.6795,
-      "step": 2152
+      "epoch": 1.858736059479554,
+      "grad_norm": 14.726641654968262,
+      "learning_rate": 8.105263157894736e-06,
+      "loss": 0.7749,
+      "step": 1000
     },
     {
       "epoch": 2.0,
-      "eval_f1": 0.8069274438342103,
-      "eval_loss": 0.6678793430328369,
-      "eval_runtime": 1131.1127,
-      "eval_samples_per_second": 7.608,
-      "eval_steps_per_second": 0.238,
-      "step": 2152
+      "eval_f1": 0.7999964392111495,
+      "eval_loss": 0.7103919982910156,
+      "eval_runtime": 22.0431,
+      "eval_samples_per_second": 390.371,
+      "eval_steps_per_second": 6.124,
+      "step": 1076
     },
     {
-      "epoch": 3.0,
-      "grad_norm": 20.788061141967773,
-      "learning_rate": 9.004184100418411e-06,
-      "loss": 0.4878,
-      "step": 3228
+      "epoch": 2.7881040892193307,
+      "grad_norm": 18.681787490844727,
+      "learning_rate": 5.717703349282297e-06,
+      "loss": 0.6165,
+      "step": 1500
     },
     {
       "epoch": 3.0,
-      "eval_f1": 0.8184543003319226,
-      "eval_loss": 0.6415576338768005,
-      "eval_runtime": 1137.7492,
-      "eval_samples_per_second": 7.563,
-      "eval_steps_per_second": 0.236,
-      "step": 3228
+      "eval_f1": 0.8113637968918935,
+      "eval_loss": 0.6536160707473755,
+      "eval_runtime": 22.0131,
+      "eval_samples_per_second": 390.904,
+      "eval_steps_per_second": 6.133,
+      "step": 1614
     },
     {
-      "epoch": 4.0,
-      "grad_norm": 31.943090438842773,
-      "learning_rate": 4.5020920502092055e-06,
-      "loss": 0.339,
-      "step": 4304
+      "epoch": 3.717472118959108,
+      "grad_norm": 15.741679191589355,
+      "learning_rate": 3.3253588516746417e-06,
+      "loss": 0.5044,
+      "step": 2000
     },
     {
       "epoch": 4.0,
-      "eval_f1": 0.8262643018421877,
-      "eval_loss": 0.6522042155265808,
-      "eval_runtime": 1113.9155,
-      "eval_samples_per_second": 7.725,
-      "eval_steps_per_second": 0.241,
-      "step": 4304
+      "eval_f1": 0.8140234226099499,
+      "eval_loss": 0.6571179628372192,
+      "eval_runtime": 22.0192,
+      "eval_samples_per_second": 390.795,
+      "eval_steps_per_second": 6.131,
+      "step": 2152
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 10.032992362976074,
-      "learning_rate": 0.0,
-      "loss": 0.226,
-      "step": 5380
+      "epoch": 4.646840148698884,
+      "grad_norm": 12.586956024169922,
+      "learning_rate": 9.330143540669858e-07,
+      "loss": 0.4217,
+      "step": 2500
     },
     {
       "epoch": 5.0,
-      "eval_f1": 0.8262395269824653,
-      "eval_loss": 0.6748126149177551,
-      "eval_runtime": 1135.4492,
-      "eval_samples_per_second": 7.578,
-      "eval_steps_per_second": 0.237,
-      "step": 5380
+      "eval_f1": 0.8154414212199751,
+      "eval_loss": 0.6651344299316406,
+      "eval_runtime": 21.8841,
+      "eval_samples_per_second": 393.209,
+      "eval_steps_per_second": 6.169,
+      "step": 2690
     },
     {
       "epoch": 5.0,
-      "step": 5380,
+      "step": 2690,
       "total_flos": 1.603929743474688e+17,
-      "train_loss": 0.5906805073904725,
-      "train_runtime": 93341.3352,
-      "train_samples_per_second": 1.844,
-      "train_steps_per_second": 0.058
+      "train_loss": 0.7819821552716223,
+      "train_runtime": 1776.2036,
+      "train_samples_per_second": 96.892,
+      "train_steps_per_second": 1.514
     }
   ],
   "logging_steps": 500,
-  "max_steps": 5380,
+  "max_steps": 2690,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 5,
   "save_steps": 500,
@@ -116,7 +116,7 @@
     }
   },
   "total_flos": 1.603929743474688e+17,
-  "train_batch_size": 32,
+  "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcd7112676da5ad151517cb6328fce426a8d218cf6913d3798f4f6f7aaf5cf68
-size 5304
+oid sha256:4199969cd5b4951b886fe0bcbeb67c79f95f400afb8f77bdfd4e0aa0554a1896
+size 5368