Upload 14 files

README.md

@@ -0,0 +1,59 @@
+---
+license: other
+library_name: peft
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
+model-index:
+- name: train_2024-05-05-10-57-50
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2024-05-05-10-57-50
+
+This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) on the Reseacrhed_data dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 2.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.10.0
+- Transformers 4.40.1
+- Pytorch 2.2.1+cu121
+- Datasets 2.19.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc828b5a7e19f979062b3f050cf3157cfee47b828bb8599721588c6f8f73e7b0
+size 4517152

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.9813084112149533,
+    "total_flos": 1.078522514767872e+16,
+    "train_loss": 2.9435044122192093,
+    "train_runtime": 1048.2079,
+    "train_samples_per_second": 1.631,
+    "train_steps_per_second": 0.101
+}

running_log.txt

@@ -0,0 +1,227 @@
+05/05/2024 11:09:07 - INFO - transformers.tokenization_utils_base - loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/tokenizer.model
+
+05/05/2024 11:09:07 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/tokenizer.json
+
+05/05/2024 11:09:07 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None
+
+05/05/2024 11:09:07 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/special_tokens_map.json
+
+05/05/2024 11:09:07 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/tokenizer_config.json
+
+05/05/2024 11:09:07 - INFO - llmtuner.data.loader - Loading dataset Reseacrhed_data.json...
+
+05/05/2024 11:09:07 - WARNING - llmtuner.data.utils - Checksum failed: missing SHA-1 hash value in dataset_info.json.
+
+05/05/2024 11:09:09 - WARNING - transformers.tokenization_utils_base - Token indices sequence length is longer than the specified maximum sequence length for this model (2387 > 2048). Running this sequence through the model will result in indexing errors
+
+05/05/2024 11:09:09 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/config.json
+
+05/05/2024 11:09:09 - INFO - transformers.configuration_utils - Model config LlamaConfig {
+  "_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+05/05/2024 11:09:09 - INFO - llmtuner.model.utils.quantization - Quantizing model to 8 bit.
+
+05/05/2024 11:09:30 - INFO - transformers.modeling_utils - loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/model.safetensors
+
+05/05/2024 11:09:30 - INFO - transformers.modeling_utils - Instantiating LlamaForCausalLM model under default dtype torch.float16.
+
+05/05/2024 11:09:30 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}
+
+
+05/05/2024 11:09:34 - INFO - transformers.modeling_utils - All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+05/05/2024 11:09:34 - INFO - transformers.modeling_utils - All the weights of LlamaForCausalLM were initialized from the model checkpoint at TinyLlama/TinyLlama-1.1B-Chat-v1.0.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+05/05/2024 11:09:35 - INFO - transformers.generation.configuration_utils - loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/generation_config.json
+
+05/05/2024 11:09:35 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 2048,
+  "pad_token_id": 0
+}
+
+
+05/05/2024 11:09:35 - INFO - llmtuner.model.utils.checkpointing - Gradient checkpointing enabled.
+
+05/05/2024 11:09:35 - INFO - llmtuner.model.utils.attention - Using torch SDPA for faster training and inference.
+
+05/05/2024 11:09:35 - INFO - llmtuner.model.adapter - Fine-tuning method: LoRA
+
+05/05/2024 11:09:35 - INFO - llmtuner.model.loader - trainable params: 1126400 || all params: 1101174784 || trainable%: 0.1023
+
+05/05/2024 11:09:35 - INFO - transformers.trainer - Using auto half precision backend
+
+05/05/2024 11:09:35 - INFO - transformers.trainer - ***** Running training *****
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Num examples = 855
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Num Epochs = 2
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Instantaneous batch size per device = 2
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Total train batch size (w. parallel, distributed & accumulation) = 16
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Gradient Accumulation steps = 8
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Total optimization steps = 106
+
+05/05/2024 11:09:35 - INFO - transformers.trainer -   Number of trainable parameters = 1,126,400
+
+05/05/2024 11:10:23 - INFO - llmtuner.extras.callbacks - {'loss': 2.9430, 'learning_rate': 1.9890e-05, 'epoch': 0.09}
+
+05/05/2024 11:11:11 - INFO - llmtuner.extras.callbacks - {'loss': 3.0203, 'learning_rate': 1.9564e-05, 'epoch': 0.19}
+
+05/05/2024 11:11:59 - INFO - llmtuner.extras.callbacks - {'loss': 2.8306, 'learning_rate': 1.9028e-05, 'epoch': 0.28}
+
+05/05/2024 11:12:48 - INFO - llmtuner.extras.callbacks - {'loss': 2.9983, 'learning_rate': 1.8294e-05, 'epoch': 0.37}
+
+05/05/2024 11:13:37 - INFO - llmtuner.extras.callbacks - {'loss': 3.0059, 'learning_rate': 1.7378e-05, 'epoch': 0.47}
+
+05/05/2024 11:14:26 - INFO - llmtuner.extras.callbacks - {'loss': 2.8450, 'learning_rate': 1.6301e-05, 'epoch': 0.56}
+
+05/05/2024 11:15:16 - INFO - llmtuner.extras.callbacks - {'loss': 2.9230, 'learning_rate': 1.5085e-05, 'epoch': 0.65}
+
+05/05/2024 11:16:05 - INFO - llmtuner.extras.callbacks - {'loss': 2.9725, 'learning_rate': 1.3758e-05, 'epoch': 0.75}
+
+05/05/2024 11:16:55 - INFO - llmtuner.extras.callbacks - {'loss': 2.9861, 'learning_rate': 1.2349e-05, 'epoch': 0.84}
+
+05/05/2024 11:17:45 - INFO - llmtuner.extras.callbacks - {'loss': 2.9897, 'learning_rate': 1.0888e-05, 'epoch': 0.93}
+
+05/05/2024 11:18:35 - INFO - llmtuner.extras.callbacks - {'loss': 2.9677, 'learning_rate': 9.4076e-06, 'epoch': 1.03}
+
+05/05/2024 11:19:24 - INFO - llmtuner.extras.callbacks - {'loss': 2.9620, 'learning_rate': 7.9402e-06, 'epoch': 1.12}
+
+05/05/2024 11:20:14 - INFO - llmtuner.extras.callbacks - {'loss': 2.9335, 'learning_rate': 6.5180e-06, 'epoch': 1.21}
+
+05/05/2024 11:21:04 - INFO - llmtuner.extras.callbacks - {'loss': 2.9682, 'learning_rate': 5.1721e-06, 'epoch': 1.31}
+
+05/05/2024 11:21:54 - INFO - llmtuner.extras.callbacks - {'loss': 2.9475, 'learning_rate': 3.9320e-06, 'epoch': 1.40}
+
+05/05/2024 11:22:43 - INFO - llmtuner.extras.callbacks - {'loss': 2.9373, 'learning_rate': 2.8249e-06, 'epoch': 1.50}
+
+05/05/2024 11:23:34 - INFO - llmtuner.extras.callbacks - {'loss': 2.9115, 'learning_rate': 1.8751e-06, 'epoch': 1.59}
+
+05/05/2024 11:24:24 - INFO - llmtuner.extras.callbacks - {'loss': 2.9237, 'learning_rate': 1.1034e-06, 'epoch': 1.68}
+
+05/05/2024 11:25:13 - INFO - llmtuner.extras.callbacks - {'loss': 2.8901, 'learning_rate': 5.2674e-07, 'epoch': 1.78}
+
+05/05/2024 11:26:03 - INFO - llmtuner.extras.callbacks - {'loss': 2.8905, 'learning_rate': 1.5769e-07, 'epoch': 1.87}
+
+05/05/2024 11:26:03 - INFO - transformers.trainer - Saving model checkpoint to saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50/checkpoint-100
+
+05/05/2024 11:26:03 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/config.json
+
+05/05/2024 11:26:03 - INFO - transformers.configuration_utils - Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+05/05/2024 11:26:03 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50/checkpoint-100/tokenizer_config.json
+
+05/05/2024 11:26:03 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50/checkpoint-100/special_tokens_map.json
+
+05/05/2024 11:26:53 - INFO - llmtuner.extras.callbacks - {'loss': 2.9548, 'learning_rate': 4.3916e-09, 'epoch': 1.96}
+
+05/05/2024 11:27:03 - INFO - transformers.trainer - 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+05/05/2024 11:27:03 - INFO - transformers.trainer - Saving model checkpoint to saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50
+
+05/05/2024 11:27:03 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-Chat-v1.0/snapshots/fe8a4ea1ffedaf415f4da2f062534de366a451e6/config.json
+
+05/05/2024 11:27:03 - INFO - transformers.configuration_utils - Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+05/05/2024 11:27:03 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50/tokenizer_config.json
+
+05/05/2024 11:27:03 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50/special_tokens_map.json
+
+05/05/2024 11:27:03 - INFO - transformers.modelcard - Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message + '\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '\\nAssistant: ' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.9813084112149533,
+    "total_flos": 1.078522514767872e+16,
+    "train_loss": 2.9435044122192093,
+    "train_runtime": 1048.2079,
+    "train_samples_per_second": 1.631,
+    "train_steps_per_second": 0.101
+}

trainer_config.yaml

@@ -0,0 +1,29 @@
+cutoff_len: 1024
+dataset: Reseacrhed_data
+dataset_dir: data
+do_train: true
+finetuning_type: lora
+flash_attn: auto
+fp16: true
+gradient_accumulation_steps: 8
+learning_rate: 2.0e-05
+logging_steps: 5
+lora_alpha: 16
+lora_dropout: 0
+lora_rank: 8
+lora_target: q_proj,v_proj
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 10000
+model_name_or_path: TinyLlama/TinyLlama-1.1B-Chat-v1.0
+num_train_epochs: 2.0
+optim: adamw_torch
+output_dir: saves/LLaMA-tiny/lora/train_2024-05-05-10-57-50
+packing: true
+per_device_train_batch_size: 2
+quantization_bit: 8
+report_to: none
+save_steps: 100
+stage: pt
+template: default
+warmup_steps: 0

trainer_log.jsonl

@@ -0,0 +1,22 @@
+{"current_steps": 5, "total_steps": 106, "loss": 2.943, "learning_rate": 1.9890401873221642e-05, "epoch": 0.09345794392523364, "percentage": 4.72, "elapsed_time": "0:00:47", "remaining_time": "0:16:06"}
+{"current_steps": 10, "total_steps": 106, "loss": 3.0203, "learning_rate": 1.9564009842765225e-05, "epoch": 0.18691588785046728, "percentage": 9.43, "elapsed_time": "0:01:35", "remaining_time": "0:15:16"}
+{"current_steps": 15, "total_steps": 106, "loss": 2.8306, "learning_rate": 1.9027978299657436e-05, "epoch": 0.2803738317757009, "percentage": 14.15, "elapsed_time": "0:02:23", "remaining_time": "0:14:32"}
+{"current_steps": 20, "total_steps": 106, "loss": 2.9983, "learning_rate": 1.829405685450202e-05, "epoch": 0.37383177570093457, "percentage": 18.87, "elapsed_time": "0:03:13", "remaining_time": "0:13:49"}
+{"current_steps": 25, "total_steps": 106, "loss": 3.0059, "learning_rate": 1.7378332790417275e-05, "epoch": 0.4672897196261682, "percentage": 23.58, "elapsed_time": "0:04:02", "remaining_time": "0:13:04"}
+{"current_steps": 30, "total_steps": 106, "loss": 2.845, "learning_rate": 1.6300878435817115e-05, "epoch": 0.5607476635514018, "percentage": 28.3, "elapsed_time": "0:04:51", "remaining_time": "0:12:18"}
+{"current_steps": 35, "total_steps": 106, "loss": 2.923, "learning_rate": 1.5085311186492206e-05, "epoch": 0.6542056074766355, "percentage": 33.02, "elapsed_time": "0:05:40", "remaining_time": "0:11:31"}
+{"current_steps": 40, "total_steps": 106, "loss": 2.9725, "learning_rate": 1.3758275821142382e-05, "epoch": 0.7476635514018691, "percentage": 37.74, "elapsed_time": "0:06:30", "remaining_time": "0:10:44"}
+{"current_steps": 45, "total_steps": 106, "loss": 2.9861, "learning_rate": 1.234886045780984e-05, "epoch": 0.8411214953271028, "percentage": 42.45, "elapsed_time": "0:07:20", "remaining_time": "0:09:56"}
+{"current_steps": 50, "total_steps": 106, "loss": 2.9897, "learning_rate": 1.0887958953229349e-05, "epoch": 0.9345794392523364, "percentage": 47.17, "elapsed_time": "0:08:10", "remaining_time": "0:09:08"}
+{"current_steps": 55, "total_steps": 106, "loss": 2.9677, "learning_rate": 9.407593721062858e-06, "epoch": 1.02803738317757, "percentage": 51.89, "elapsed_time": "0:08:59", "remaining_time": "0:08:20"}
+{"current_steps": 60, "total_steps": 106, "loss": 2.962, "learning_rate": 7.940213812589018e-06, "epoch": 1.1214953271028036, "percentage": 56.6, "elapsed_time": "0:09:49", "remaining_time": "0:07:31"}
+{"current_steps": 65, "total_steps": 106, "loss": 2.9335, "learning_rate": 6.517983645656014e-06, "epoch": 1.2149532710280373, "percentage": 61.32, "elapsed_time": "0:10:39", "remaining_time": "0:06:43"}
+{"current_steps": 70, "total_steps": 106, "loss": 2.9682, "learning_rate": 5.172077972692553e-06, "epoch": 1.308411214953271, "percentage": 66.04, "elapsed_time": "0:11:29", "remaining_time": "0:05:54"}
+{"current_steps": 75, "total_steps": 106, "loss": 2.9475, "learning_rate": 3.931998541814069e-06, "epoch": 1.4018691588785046, "percentage": 70.75, "elapsed_time": "0:12:18", "remaining_time": "0:05:05"}
+{"current_steps": 80, "total_steps": 106, "loss": 2.9373, "learning_rate": 2.8249274295566863e-06, "epoch": 1.4953271028037383, "percentage": 75.47, "elapsed_time": "0:13:08", "remaining_time": "0:04:16"}
+{"current_steps": 85, "total_steps": 106, "loss": 2.9115, "learning_rate": 1.875131219943187e-06, "epoch": 1.588785046728972, "percentage": 80.19, "elapsed_time": "0:13:58", "remaining_time": "0:03:27"}
+{"current_steps": 90, "total_steps": 106, "loss": 2.9237, "learning_rate": 1.1034290900525279e-06, "epoch": 1.6822429906542056, "percentage": 84.91, "elapsed_time": "0:14:48", "remaining_time": "0:02:37"}
+{"current_steps": 95, "total_steps": 106, "loss": 2.8901, "learning_rate": 5.267364614580861e-07, "epoch": 1.7757009345794392, "percentage": 89.62, "elapsed_time": "0:15:38", "remaining_time": "0:01:48"}
+{"current_steps": 100, "total_steps": 106, "loss": 2.8905, "learning_rate": 1.5769422052403172e-07, "epoch": 1.8691588785046729, "percentage": 94.34, "elapsed_time": "0:16:28", "remaining_time": "0:00:59"}
+{"current_steps": 105, "total_steps": 106, "loss": 2.9548, "learning_rate": 4.39163491205652e-09, "epoch": 1.9626168224299065, "percentage": 99.06, "elapsed_time": "0:17:18", "remaining_time": "0:00:09"}
+{"current_steps": 106, "total_steps": 106, "epoch": 1.9813084112149533, "percentage": 100.0, "elapsed_time": "0:17:28", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,177 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9813084112149533,
+  "eval_steps": 500,
+  "global_step": 106,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.09345794392523364,
+      "grad_norm": 0.37823450565338135,
+      "learning_rate": 1.9890401873221642e-05,
+      "loss": 2.943,
+      "step": 5
+    },
+    {
+      "epoch": 0.18691588785046728,
+      "grad_norm": 0.3274012804031372,
+      "learning_rate": 1.9564009842765225e-05,
+      "loss": 3.0203,
+      "step": 10
+    },
+    {
+      "epoch": 0.2803738317757009,
+      "grad_norm": 0.2663849890232086,
+      "learning_rate": 1.9027978299657436e-05,
+      "loss": 2.8306,
+      "step": 15
+    },
+    {
+      "epoch": 0.37383177570093457,
+      "grad_norm": 0.2998671233654022,
+      "learning_rate": 1.829405685450202e-05,
+      "loss": 2.9983,
+      "step": 20
+    },
+    {
+      "epoch": 0.4672897196261682,
+      "grad_norm": 0.30643677711486816,
+      "learning_rate": 1.7378332790417275e-05,
+      "loss": 3.0059,
+      "step": 25
+    },
+    {
+      "epoch": 0.5607476635514018,
+      "grad_norm": 0.3167150318622589,
+      "learning_rate": 1.6300878435817115e-05,
+      "loss": 2.845,
+      "step": 30
+    },
+    {
+      "epoch": 0.6542056074766355,
+      "grad_norm": 0.29322898387908936,
+      "learning_rate": 1.5085311186492206e-05,
+      "loss": 2.923,
+      "step": 35
+    },
+    {
+      "epoch": 0.7476635514018691,
+      "grad_norm": 0.3546823263168335,
+      "learning_rate": 1.3758275821142382e-05,
+      "loss": 2.9725,
+      "step": 40
+    },
+    {
+      "epoch": 0.8411214953271028,
+      "grad_norm": 0.35268881916999817,
+      "learning_rate": 1.234886045780984e-05,
+      "loss": 2.9861,
+      "step": 45
+    },
+    {
+      "epoch": 0.9345794392523364,
+      "grad_norm": 0.3079037666320801,
+      "learning_rate": 1.0887958953229349e-05,
+      "loss": 2.9897,
+      "step": 50
+    },
+    {
+      "epoch": 1.02803738317757,
+      "grad_norm": 0.33415308594703674,
+      "learning_rate": 9.407593721062858e-06,
+      "loss": 2.9677,
+      "step": 55
+    },
+    {
+      "epoch": 1.1214953271028036,
+      "grad_norm": 0.3114282190799713,
+      "learning_rate": 7.940213812589018e-06,
+      "loss": 2.962,
+      "step": 60
+    },
+    {
+      "epoch": 1.2149532710280373,
+      "grad_norm": 0.3043849766254425,
+      "learning_rate": 6.517983645656014e-06,
+      "loss": 2.9335,
+      "step": 65
+    },
+    {
+      "epoch": 1.308411214953271,
+      "grad_norm": 0.35792258381843567,
+      "learning_rate": 5.172077972692553e-06,
+      "loss": 2.9682,
+      "step": 70
+    },
+    {
+      "epoch": 1.4018691588785046,
+      "grad_norm": 0.30202534794807434,
+      "learning_rate": 3.931998541814069e-06,
+      "loss": 2.9475,
+      "step": 75
+    },
+    {
+      "epoch": 1.4953271028037383,
+      "grad_norm": 0.40600359439849854,
+      "learning_rate": 2.8249274295566863e-06,
+      "loss": 2.9373,
+      "step": 80
+    },
+    {
+      "epoch": 1.588785046728972,
+      "grad_norm": 0.31094491481781006,
+      "learning_rate": 1.875131219943187e-06,
+      "loss": 2.9115,
+      "step": 85
+    },
+    {
+      "epoch": 1.6822429906542056,
+      "grad_norm": 0.3808990716934204,
+      "learning_rate": 1.1034290900525279e-06,
+      "loss": 2.9237,
+      "step": 90
+    },
+    {
+      "epoch": 1.7757009345794392,
+      "grad_norm": 0.33799558877944946,
+      "learning_rate": 5.267364614580861e-07,
+      "loss": 2.8901,
+      "step": 95
+    },
+    {
+      "epoch": 1.8691588785046729,
+      "grad_norm": 0.3236595690250397,
+      "learning_rate": 1.5769422052403172e-07,
+      "loss": 2.8905,
+      "step": 100
+    },
+    {
+      "epoch": 1.9626168224299065,
+      "grad_norm": 0.3427414000034332,
+      "learning_rate": 4.39163491205652e-09,
+      "loss": 2.9548,
+      "step": 105
+    },
+    {
+      "epoch": 1.9813084112149533,
+      "step": 106,
+      "total_flos": 1.078522514767872e+16,
+      "train_loss": 2.9435044122192093,
+      "train_runtime": 1048.2079,
+      "train_samples_per_second": 1.631,
+      "train_steps_per_second": 0.101
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 106,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "total_flos": 1.078522514767872e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c8b10cfcdadcb4d9c13a2f7dc44a0e53f10d1b92bcbc2b09c784742ad8b6290
+size 5176