Model save

Qwen2.5-1.5B-Open-R1-GRPO.log

@@ -1,7 +1,7 @@
-2025-02-17 14:24:25 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1 distributed training: True, 16-bits training: False
-2025-02-17 14:24:25 - INFO - __main__ - Model parameters ModelConfig(model_name_or_path='Qwen/Qwen2.5-1.5B-Instruct', model_revision='main', torch_dtype='bfloat16', trust_remote_code=False, attn_implementation='flash_attention_2', use_peft=False, lora_r=16, lora_alpha=32, lora_dropout=0.05, lora_target_modules=None, lora_modules_to_save=None, lora_task_type='CAUSAL_LM', use_rslora=False, load_in_8bit=False, load_in_4bit=False, bnb_4bit_quant_type='nf4', use_bnb_nested_quant=False)
-2025-02-17 14:24:25 - INFO - __main__ - Script parameters GRPOScriptArguments(dataset_name='AI-MO/NuminaMath-TIR', dataset_config=None, dataset_train_split='train', dataset_test_split='test', gradient_checkpointing_use_reentrant=False, ignore_bias_buffers=False, reward_funcs=['accuracy', 'format'], cosine_min_value_wrong=0.0, cosine_max_value_wrong=-0.5, cosine_min_value_correct=0.5, cosine_max_value_correct=1.0, cosine_max_len=1000, repetition_n_grams=3, repetition_max_penalty=-1.0)
-2025-02-17 14:24:25 - INFO - __main__ - Training parameters GRPOConfig(
+2025-02-18 14:33:03 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1 distributed training: True, 16-bits training: False
+2025-02-18 14:33:03 - INFO - __main__ - Model parameters ModelConfig(model_name_or_path='Qwen/Qwen2.5-1.5B-Instruct', model_revision='main', torch_dtype='bfloat16', trust_remote_code=False, attn_implementation='flash_attention_2', use_peft=False, lora_r=16, lora_alpha=32, lora_dropout=0.05, lora_target_modules=None, lora_modules_to_save=None, lora_task_type='CAUSAL_LM', use_rslora=False, load_in_8bit=False, load_in_4bit=False, bnb_4bit_quant_type='nf4', use_bnb_nested_quant=False)
+2025-02-18 14:33:03 - INFO - __main__ - Script parameters GRPOScriptArguments(dataset_name='AI-MO/NuminaMath-TIR', dataset_config=None, dataset_train_split='train', dataset_test_split='test', gradient_checkpointing_use_reentrant=False, ignore_bias_buffers=False, reward_funcs=['accuracy', 'format'], cosine_min_value_wrong=0.0, cosine_max_value_wrong=-0.5, cosine_min_value_correct=0.5, cosine_max_value_correct=1.0, cosine_max_len=1000, repetition_n_grams=3, repetition_max_penalty=-1.0)
+2025-02-18 14:33:03 - INFO - __main__ - Training parameters GRPOConfig(
 _n_gpu=1,
 accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False},
 adafactor=False,
@@ -80,7 +80,7 @@ log_completions=False,
 log_level=info,
 log_level_replica=warning,
 log_on_each_node=True,
-logging_dir=output/Qwen2.5-1.5B-Open-R1-GRPO/runs/Feb17_14-24-25_smilelab-a6000,
+logging_dir=output/Qwen2.5-1.5B-Open-R1-GRPO/runs/Feb18_14-33-03_smilelab-a6000,
 logging_first_step=False,
 logging_nan_inf_filter=True,
 logging_steps=5,
@@ -158,14 +158,183 @@ warmup_ratio=0.1,
 warmup_steps=0,
 weight_decay=0.0,
 )
-2025-02-17 14:24:25 - WARNING - __main__ - Process rank: 2, device: cuda:2, n_gpu: 1 distributed training: True, 16-bits training: False
-2025-02-17 14:24:25 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1 distributed training: True, 16-bits training: False
-2025-02-17 14:24:26 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
-2025-02-17 14:24:26 - INFO - datasets.info - Loading Dataset info from /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd
-2025-02-17 14:24:26 - INFO - datasets.builder - Found cached dataset numina_math-tir (/home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd)
-2025-02-17 14:24:26 - INFO - datasets.info - Loading Dataset info from /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd
-2025-02-17 14:24:26 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd/cache-42e22e57818641cd.arrow
-2025-02-17 14:24:26 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd/cache-be70039808739fd6.arrow
-2025-02-17 14:24:26 - INFO - __main__ - *** Initializing model kwargs ***
-2025-02-17 14:24:50 - INFO - __main__ - *** Train ***
-2025-02-18 05:27:58 - INFO - __main__ - *** Save model ***
+2025-02-18 14:33:03 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1 distributed training: True, 16-bits training: False
+2025-02-18 14:33:03 - WARNING - __main__ - Process rank: 2, device: cuda:2, n_gpu: 1 distributed training: True, 16-bits training: False
+2025-02-18 14:33:04 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+2025-02-18 14:33:04 - INFO - datasets.info - Loading Dataset info from /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd
+2025-02-18 14:33:04 - INFO - datasets.builder - Found cached dataset numina_math-tir (/home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd)
+2025-02-18 14:33:04 - INFO - datasets.info - Loading Dataset info from /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd
+2025-02-18 14:33:04 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd/cache-42e22e57818641cd.arrow
+2025-02-18 14:33:04 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd/cache-be70039808739fd6.arrow
+2025-02-18 14:33:04 - INFO - __main__ - *** Initializing model kwargs ***
+2025-02-18 14:33:31 - INFO - __main__ - *** Train ***
+2025-02-18 14:39:05 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1 distributed training: True, 16-bits training: False
+2025-02-18 14:39:05 - INFO - __main__ - Model parameters ModelConfig(model_name_or_path='Qwen/Qwen2.5-1.5B-Instruct', model_revision='main', torch_dtype='bfloat16', trust_remote_code=False, attn_implementation='flash_attention_2', use_peft=False, lora_r=16, lora_alpha=32, lora_dropout=0.05, lora_target_modules=None, lora_modules_to_save=None, lora_task_type='CAUSAL_LM', use_rslora=False, load_in_8bit=False, load_in_4bit=False, bnb_4bit_quant_type='nf4', use_bnb_nested_quant=False)
+2025-02-18 14:39:05 - INFO - __main__ - Script parameters GRPOScriptArguments(dataset_name='AI-MO/NuminaMath-TIR', dataset_config=None, dataset_train_split='train', dataset_test_split='test', gradient_checkpointing_use_reentrant=False, ignore_bias_buffers=False, reward_funcs=['accuracy', 'format'], cosine_min_value_wrong=0.0, cosine_max_value_wrong=-0.5, cosine_min_value_correct=0.5, cosine_max_value_correct=1.0, cosine_max_len=1000, repetition_n_grams=3, repetition_max_penalty=-1.0)
+2025-02-18 14:39:05 - INFO - __main__ - Training parameters GRPOConfig(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+average_tokens_across_devices=False,
+batch_eval_metrics=False,
+benchmarks=[],
+beta=0.04,
+bf16=True,
+bf16_full_eval=False,
+callbacks=[],
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=True,
+do_predict=False,
+do_train=False,
+ds3_gather_for_generation=True,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_on_start=False,
+eval_steps=100,
+eval_strategy=IntervalStrategy.STEPS,
+eval_use_gather_object=False,
+evaluation_strategy=None,
+fp16=False,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs={'use_reentrant': False},
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=Qwen2.5-1.5B-Open-R1-GRPO,
+hub_model_revision=main,
+hub_private_repo=None,
+hub_strategy=HubStrategy.EVERY_SAVE,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_for_metrics=[],
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=2e-05,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_completions=False,
+log_level=info,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/Qwen2.5-1.5B-Open-R1-GRPO/runs/Feb18_14-39-05_smilelab-a6000,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=5,
+logging_strategy=IntervalStrategy.STEPS,
+lr_scheduler_kwargs={},
+lr_scheduler_type=SchedulerType.COSINE,
+max_completion_length=1024,
+max_grad_norm=1.0,
+max_prompt_length=512,
+max_steps=-1,
+metric_for_best_model=None,
+model_init_kwargs=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_generations=2,
+num_train_epochs=1,
+optim=OptimizerNames.ADAMW_TORCH,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/Qwen2.5-1.5B-Open-R1-GRPO,
+overwrite_hub_revision=False,
+overwrite_output_dir=True,
+past_index=-1,
+per_device_eval_batch_size=32,
+per_device_train_batch_size=16,
+prediction_loss_only=False,
+push_to_hub=True,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_revision=False,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+ref_model_mixup_alpha=0.9,
+ref_model_sync_steps=64,
+remove_unused_columns=False,
+report_to=['wandb'],
+restore_callback_states_from_checkpoint=False,
+resume_from_checkpoint=None,
+reward_weights=None,
+run_name=output/Qwen2.5-1.5B-Open-R1-GRPO,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=500,
+save_strategy=SaveStrategy.NO,
+save_total_limit=None,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+sync_ref_model=False,
+system_prompt=None,
+temperature=0.9,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torch_empty_cache_steps=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_liger_kernel=False,
+use_mps_device=False,
+use_vllm=True,
+vllm_device=auto,
+vllm_dtype=auto,
+vllm_gpu_memory_utilization=0.7,
+vllm_max_model_len=None,
+wandb_entity=None,
+wandb_project=None,
+warmup_ratio=0.1,
+warmup_steps=0,
+weight_decay=0.0,
+)
+2025-02-18 14:39:05 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1 distributed training: True, 16-bits training: False
+2025-02-18 14:39:07 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+2025-02-18 14:39:07 - INFO - datasets.info - Loading Dataset info from /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd
+2025-02-18 14:39:07 - INFO - datasets.builder - Found cached dataset numina_math-tir (/home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd)
+2025-02-18 14:39:07 - INFO - datasets.info - Loading Dataset info from /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd
+2025-02-18 14:39:07 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd/cache-42e22e57818641cd.arrow
+2025-02-18 14:39:07 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/mingyuan/.cache/huggingface/datasets/AI-MO___numina_math-tir/default/0.0.0/77a91d7b7a1a98ac4b1beb7d86c09d156b935dcd/cache-be70039808739fd6.arrow
+2025-02-18 14:39:07 - INFO - __main__ - *** Initializing model kwargs ***
+2025-02-18 14:39:33 - INFO - __main__ - *** Train ***
+2025-02-19 03:54:32 - INFO - __main__ - *** Save model ***

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/no-exsit/huggingface/runs/2dcu6uam) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/no-exsit/huggingface/runs/ya30e8vy) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.1736428747392735,
-    "train_runtime": 54182.9853,
+    "train_loss": 27.009561450669516,
+    "train_runtime": 47693.8258,
     "train_samples": 72441,
-    "train_samples_per_second": 1.337,
-    "train_steps_per_second": 0.005
+    "train_samples_per_second": 1.519,
+    "train_steps_per_second": 0.006
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f560b0bd324be829118baf66fc7c7aeaf451d3a837a09bb002084221d0d76b3e
-size 3554214752
+oid sha256:48021ab932819ba74e1bce8a7cb87f288cbc1346247da4002d7bb453f1e52516
+size 3087467144

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.1736428747392735,
-    "train_runtime": 54182.9853,
+    "train_loss": 27.009561450669516,
+    "train_runtime": 47693.8258,
     "train_samples": 72441,
-    "train_samples_per_second": 1.337,
-    "train_steps_per_second": 0.005
+    "train_samples_per_second": 1.519,
+    "train_steps_per_second": 0.006
 }

trainer_state.json

@@ -9,775 +9,775 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 490.6283988952637,
+      "completion_length": 404.396875,
       "epoch": 0.0176678445229682,
-      "grad_norm": 1.3560127019882202,
-      "kl": 0.0019670963287353516,
+      "grad_norm": 0.785922280854803,
+      "kl": 0.00035384297370910645,
       "learning_rate": 3.448275862068966e-06,
-      "loss": 0.0001,
-      "reward": 0.21250000642612576,
-      "reward_std": 0.21599052464589477,
-      "rewards/accuracy_reward": 0.05130208469927311,
-      "rewards/format_reward": 0.16119792133104055,
+      "loss": 0.0,
+      "reward": 0.5984375,
+      "reward_std": 0.31488348115235565,
+      "rewards/accuracy_reward": 0.166015625,
+      "rewards/format_reward": 0.432421875,
       "step": 5
     },
     {
-      "completion_length": 534.7315277099609,
+      "completion_length": 410.702734375,
       "epoch": 0.0353356890459364,
-      "grad_norm": 82.46902465820312,
-      "kl": 0.11877202987670898,
+      "grad_norm": 0.3143792157068426,
+      "kl": 0.13354759216308593,
       "learning_rate": 6.896551724137932e-06,
-      "loss": 0.0048,
-      "reward": 0.10625000358559192,
-      "reward_std": 0.16045468132942914,
-      "rewards/accuracy_reward": 0.022395833861082792,
-      "rewards/format_reward": 0.0838541688863188,
+      "loss": 0.0054,
+      "reward": 0.57265625,
+      "reward_std": 0.30052037397399545,
+      "rewards/accuracy_reward": 0.160546875,
+      "rewards/format_reward": 0.412109375,
       "step": 10
     },
     {
-      "completion_length": 527.6500186920166,
+      "completion_length": 400.45234375,
       "epoch": 0.053003533568904596,
-      "grad_norm": 0.742699146270752,
-      "kl": 0.10464630126953126,
+      "grad_norm": 0.6724373940493742,
+      "kl": 0.024787521362304686,
       "learning_rate": 1.0344827586206898e-05,
-      "loss": 0.0042,
-      "reward": 0.1148437530733645,
-      "reward_std": 0.16899234298616647,
-      "rewards/accuracy_reward": 0.025000000628642736,
-      "rewards/format_reward": 0.08984375256113708,
+      "loss": 0.001,
+      "reward": 0.594921875,
+      "reward_std": 0.3065970725379884,
+      "rewards/accuracy_reward": 0.153515625,
+      "rewards/format_reward": 0.44140625,
       "step": 15
     },
     {
-      "completion_length": 534.8018405914306,
+      "completion_length": 395.24921875,
       "epoch": 0.0706713780918728,
-      "grad_norm": 11.84140682220459,
-      "kl": 0.17176666259765624,
+      "grad_norm": 0.6026471879082274,
+      "kl": 0.3488151550292969,
       "learning_rate": 1.3793103448275863e-05,
-      "loss": 0.0069,
-      "reward": 0.10625000288709999,
-      "reward_std": 0.14975022254511713,
-      "rewards/accuracy_reward": 0.026822917419485746,
-      "rewards/format_reward": 0.07942708525806666,
+      "loss": 0.0139,
+      "reward": 0.613671875,
+      "reward_std": 0.3264844523742795,
+      "rewards/accuracy_reward": 0.168359375,
+      "rewards/format_reward": 0.4453125,
       "step": 20
     },
     {
-      "completion_length": 536.3153781890869,
+      "completion_length": 398.496484375,
       "epoch": 0.08833922261484099,
-      "grad_norm": 9.572907447814941,
-      "kl": 0.37183837890625,
+      "grad_norm": 0.46593386880605636,
+      "kl": 0.1805694580078125,
       "learning_rate": 1.7241379310344828e-05,
-      "loss": 0.0149,
-      "reward": 0.10234375295694917,
-      "reward_std": 0.14782514404505492,
-      "rewards/accuracy_reward": 0.02083333386108279,
-      "rewards/format_reward": 0.0815104188863188,
+      "loss": 0.0072,
+      "reward": 0.61484375,
+      "reward_std": 0.2883669765666127,
+      "rewards/accuracy_reward": 0.158203125,
+      "rewards/format_reward": 0.456640625,
       "step": 25
     },
     {
-      "completion_length": 539.7317886352539,
+      "completion_length": 401.319921875,
       "epoch": 0.10600706713780919,
-      "grad_norm": 20.72088050842285,
-      "kl": 213.514892578125,
+      "grad_norm": 0.5247335452215227,
+      "kl": 0.145611572265625,
       "learning_rate": 1.999923511388017e-05,
-      "loss": 8.5621,
-      "reward": 0.11822917042300105,
-      "reward_std": 0.16361206490546465,
-      "rewards/accuracy_reward": 0.025520833930931984,
-      "rewards/format_reward": 0.09270833609625698,
+      "loss": 0.0058,
+      "reward": 0.61328125,
+      "reward_std": 0.2905766863375902,
+      "rewards/accuracy_reward": 0.16484375,
+      "rewards/format_reward": 0.4484375,
       "step": 30
     },
     {
-      "completion_length": 532.9531394958497,
+      "completion_length": 402.28671875,
       "epoch": 0.12367491166077739,
-      "grad_norm": 4.47783088684082,
-      "kl": 1.0462158203125,
+      "grad_norm": 1.3201550928400456,
+      "kl": 0.15986328125,
       "learning_rate": 1.9972476383747748e-05,
-      "loss": 0.0419,
-      "reward": 0.1104166700039059,
-      "reward_std": 0.16090573472902178,
-      "rewards/accuracy_reward": 0.024218750605359673,
-      "rewards/format_reward": 0.08619791911914945,
+      "loss": 0.0064,
+      "reward": 0.591796875,
+      "reward_std": 0.29886309332214295,
+      "rewards/accuracy_reward": 0.1515625,
+      "rewards/format_reward": 0.440234375,
       "step": 35
     },
     {
-      "completion_length": 531.2823101043701,
+      "completion_length": 400.734765625,
       "epoch": 0.1413427561837456,
-      "grad_norm": 0.9901871085166931,
-      "kl": 2.563671875,
+      "grad_norm": 0.4543361752537872,
+      "kl": 0.293060302734375,
       "learning_rate": 1.9907590277344582e-05,
-      "loss": 0.1025,
-      "reward": 0.09843750263098627,
-      "reward_std": 0.14138951590284704,
-      "rewards/accuracy_reward": 0.01979166711680591,
-      "rewards/format_reward": 0.07864583539776504,
+      "loss": 0.0117,
+      "reward": 0.60703125,
+      "reward_std": 0.31156891826540234,
+      "rewards/accuracy_reward": 0.166015625,
+      "rewards/format_reward": 0.441015625,
       "step": 40
     },
     {
-      "completion_length": 524.4414211273194,
+      "completion_length": 402.326953125,
       "epoch": 0.15901060070671377,
-      "grad_norm": 0.5192703604698181,
-      "kl": 0.683203125,
+      "grad_norm": 1.8083450201171443,
+      "kl": 0.539990234375,
       "learning_rate": 1.9804824871166254e-05,
-      "loss": 0.0274,
-      "reward": 0.10703125330619515,
-      "reward_std": 0.152545028924942,
-      "rewards/accuracy_reward": 0.02161458383779973,
-      "rewards/format_reward": 0.0854166692122817,
+      "loss": 0.0216,
+      "reward": 0.59609375,
+      "reward_std": 0.3016252293717116,
+      "rewards/accuracy_reward": 0.159765625,
+      "rewards/format_reward": 0.436328125,
       "step": 45
     },
     {
-      "completion_length": 538.3731964111328,
+      "completion_length": 399.735546875,
       "epoch": 0.17667844522968199,
-      "grad_norm": 6.639949798583984,
-      "kl": 0.9129150390625,
+      "grad_norm": 0.6440098529655808,
+      "kl": 36.3084228515625,
       "learning_rate": 1.9664573064143604e-05,
-      "loss": 0.0365,
-      "reward": 0.10781250279396773,
-      "reward_std": 0.15840874193236232,
-      "rewards/accuracy_reward": 0.024218750512227417,
-      "rewards/format_reward": 0.08359375253785402,
+      "loss": 1.4539,
+      "reward": 0.6109375,
+      "reward_std": 0.3060446453746408,
+      "rewards/accuracy_reward": 0.162109375,
+      "rewards/format_reward": 0.448828125,
       "step": 50
     },
     {
-      "completion_length": 521.2586082458496,
+      "completion_length": 396.709375,
       "epoch": 0.19434628975265017,
-      "grad_norm": 4.453002452850342,
-      "kl": 0.474462890625,
+      "grad_norm": 0.949235195273066,
+      "kl": 18740.40302734375,
       "learning_rate": 1.948737107548771e-05,
-      "loss": 0.019,
-      "reward": 0.10729166965465993,
-      "reward_std": 0.15750663112848998,
-      "rewards/accuracy_reward": 0.023177084024064244,
-      "rewards/format_reward": 0.08411458586342632,
+      "loss": 748.2533,
+      "reward": 0.61640625,
+      "reward_std": 0.3193028993904591,
+      "rewards/accuracy_reward": 0.165625,
+      "rewards/format_reward": 0.45078125,
       "step": 55
     },
     {
-      "completion_length": 549.339599609375,
+      "completion_length": 410.02734375,
       "epoch": 0.21201413427561838,
-      "grad_norm": 0.5541434288024902,
-      "kl": 0.66484375,
+      "grad_norm": 2.007276336933817,
+      "kl": 1.0031005859375,
       "learning_rate": 1.9273896394584103e-05,
-      "loss": 0.0266,
-      "reward": 0.09843750281725079,
-      "reward_std": 0.14917831011116506,
-      "rewards/accuracy_reward": 0.021875000582076608,
-      "rewards/format_reward": 0.07656250218860805,
+      "loss": 0.0401,
+      "reward": 0.59609375,
+      "reward_std": 0.2949961026199162,
+      "rewards/accuracy_reward": 0.148828125,
+      "rewards/format_reward": 0.447265625,
       "step": 60
     },
     {
-      "completion_length": 529.354443359375,
+      "completion_length": 405.175390625,
       "epoch": 0.22968197879858657,
-      "grad_norm": 0.7464480400085449,
-      "kl": 1.283349609375,
+      "grad_norm": 0.7554540760672372,
+      "kl": 1.448046875,
       "learning_rate": 1.9024965190774262e-05,
-      "loss": 0.0513,
-      "reward": 0.11223958733025938,
-      "reward_std": 0.16225890163332224,
-      "rewards/accuracy_reward": 0.02656250069849193,
-      "rewards/format_reward": 0.08567708532791585,
+      "loss": 0.0579,
+      "reward": 0.60546875,
+      "reward_std": 0.297205812856555,
+      "rewards/accuracy_reward": 0.15546875,
+      "rewards/format_reward": 0.45,
       "step": 65
     },
     {
-      "completion_length": 525.3435066223144,
+      "completion_length": 403.630859375,
       "epoch": 0.24734982332155478,
-      "grad_norm": 0.5229907631874084,
-      "kl": 0.46806640625,
+      "grad_norm": 0.7209308900049199,
+      "kl": 19355.033984375,
       "learning_rate": 1.8741529192927528e-05,
-      "loss": 0.0187,
-      "reward": 0.10755208674818277,
-      "reward_std": 0.1623797613196075,
-      "rewards/accuracy_reward": 0.02031250053551048,
-      "rewards/format_reward": 0.08723958565387875,
+      "loss": 776.0688,
+      "reward": 0.6125,
+      "reward_std": 0.30714950021356346,
+      "rewards/accuracy_reward": 0.165234375,
+      "rewards/format_reward": 0.447265625,
       "step": 70
     },
     {
-      "completion_length": 539.2466316223145,
+      "completion_length": 398.434375,
       "epoch": 0.26501766784452296,
-      "grad_norm": 0.9813037514686584,
-      "kl": 0.8671142578125,
+      "grad_norm": 0.5881139318633425,
+      "kl": 1.331982421875,
       "learning_rate": 1.8424672050733577e-05,
-      "loss": 0.0347,
-      "reward": 0.10989583660848438,
-      "reward_std": 0.15377733251079917,
-      "rewards/accuracy_reward": 0.02734375058207661,
-      "rewards/format_reward": 0.08255208577029408,
+      "loss": 0.0533,
+      "reward": 0.61484375,
+      "reward_std": 0.2994155207648873,
+      "rewards/accuracy_reward": 0.158203125,
+      "rewards/format_reward": 0.456640625,
       "step": 75
     },
     {
-      "completion_length": 544.274755859375,
+      "completion_length": 408.6515625,
       "epoch": 0.2826855123674912,
-      "grad_norm": 1.2748777866363525,
-      "kl": 0.545849609375,
+      "grad_norm": 2.6274469054608143,
+      "kl": 1.180029296875,
       "learning_rate": 1.8075605191627242e-05,
-      "loss": 0.0218,
-      "reward": 0.10208333632908762,
-      "reward_std": 0.15401905188336967,
-      "rewards/accuracy_reward": 0.019270833837799727,
-      "rewards/format_reward": 0.08281250209547579,
+      "loss": 0.0472,
+      "reward": 0.607421875,
+      "reward_std": 0.2811854241415858,
+      "rewards/accuracy_reward": 0.164453125,
+      "rewards/format_reward": 0.44296875,
       "step": 80
     },
     {
-      "completion_length": 536.8455909729004,
+      "completion_length": 413.64296875,
       "epoch": 0.3003533568904594,
-      "grad_norm": 0.4829046130180359,
-      "kl": 0.37125244140625,
+      "grad_norm": 0.6738788725405221,
+      "kl": 11.340185546875,
       "learning_rate": 1.7695663189185703e-05,
-      "loss": 0.0149,
-      "reward": 0.11171875377185643,
-      "reward_std": 0.1631610095500946,
-      "rewards/accuracy_reward": 0.026822917349636554,
-      "rewards/format_reward": 0.08489583544433117,
+      "loss": 0.4546,
+      "reward": 0.601953125,
+      "reward_std": 0.30549221779219804,
+      "rewards/accuracy_reward": 0.179296875,
+      "rewards/format_reward": 0.42265625,
       "step": 85
     },
     {
-      "completion_length": 526.0974117279053,
+      "completion_length": 413.73359375,
       "epoch": 0.31802120141342755,
-      "grad_norm": 4.6327128410339355,
-      "kl": 1.1112548828125,
+      "grad_norm": 1.3105277713530405,
+      "kl": 1.447265625,
       "learning_rate": 1.7286298660705877e-05,
-      "loss": 0.0445,
-      "reward": 0.09973958632908761,
-      "reward_std": 0.14544901056215168,
-      "rewards/accuracy_reward": 0.023958333861082794,
-      "rewards/format_reward": 0.07578125221189111,
+      "loss": 0.0579,
+      "reward": 0.60390625,
+      "reward_std": 0.30604464691132305,
+      "rewards/accuracy_reward": 0.1609375,
+      "rewards/format_reward": 0.44296875,
       "step": 90
     },
     {
-      "completion_length": 524.5125148773193,
+      "completion_length": 404.303515625,
       "epoch": 0.33568904593639576,
-      "grad_norm": 0.6081247925758362,
-      "kl": 0.51201171875,
+      "grad_norm": 0.7623390157660208,
+      "kl": 1.0364013671875,
       "learning_rate": 1.6849076713469914e-05,
-      "loss": 0.0205,
-      "reward": 0.11250000328291207,
-      "reward_std": 0.15795768704265356,
-      "rewards/accuracy_reward": 0.02968750074505806,
-      "rewards/format_reward": 0.08281250183936209,
+      "loss": 0.0415,
+      "reward": 0.601171875,
+      "reward_std": 0.2955485317390412,
+      "rewards/accuracy_reward": 0.16171875,
+      "rewards/format_reward": 0.439453125,
       "step": 95
     },
     {
-      "completion_length": 531.0093910217286,
+      "completion_length": 413.7109375,
       "epoch": 0.35335689045936397,
-      "grad_norm": 1.5366660356521606,
-      "kl": 0.503955078125,
+      "grad_norm": 1.0600792188331805,
+      "kl": 2.1205810546875,
       "learning_rate": 1.6385668960932143e-05,
-      "loss": 0.0202,
-      "reward": 0.09583333565387875,
-      "reward_std": 0.13687896616756917,
-      "rewards/accuracy_reward": 0.01875000037252903,
-      "rewards/format_reward": 0.07708333500195294,
+      "loss": 0.0849,
+      "reward": 0.572265625,
+      "reward_std": 0.3088067832402885,
+      "rewards/accuracy_reward": 0.1546875,
+      "rewards/format_reward": 0.417578125,
       "step": 100
     },
     {
       "epoch": 0.35335689045936397,
-      "eval_completion_length": 563.7786560058594,
-      "eval_kl": 0.3427734375,
-      "eval_loss": 0.013146836310625076,
-      "eval_reward": 0.13541666977107525,
-      "eval_reward_std": 0.20748525112867355,
-      "eval_rewards/accuracy_reward": 0.023437500465661287,
-      "eval_rewards/format_reward": 0.1119791679084301,
-      "eval_runtime": 71.5766,
-      "eval_samples_per_second": 1.383,
-      "eval_steps_per_second": 0.028,
+      "eval_completion_length": 405.1893997192383,
+      "eval_kl": 1.361328125,
+      "eval_loss": 0.05589722469449043,
+      "eval_reward": 0.61328125,
+      "eval_reward_std": 0.2927863895893097,
+      "eval_rewards/accuracy_reward": 0.140625,
+      "eval_rewards/format_reward": 0.47265625,
+      "eval_runtime": 62.5177,
+      "eval_samples_per_second": 1.584,
+      "eval_steps_per_second": 0.032,
       "step": 100
     },
     {
-      "completion_length": 539.4177280426026,
+      "completion_length": 397.241015625,
       "epoch": 0.3710247349823322,
-      "grad_norm": 0.2839926481246948,
-      "kl": 0.41923828125,
+      "grad_norm": 1.7180085275135002,
+      "kl": 2.092529296875,
       "learning_rate": 1.5897847131705194e-05,
-      "loss": 0.0168,
-      "reward": 0.10520833623595535,
-      "reward_std": 0.15119186379015445,
-      "rewards/accuracy_reward": 0.02630208395421505,
-      "rewards/format_reward": 0.0789062523515895,
+      "loss": 0.0837,
+      "reward": 0.6046875,
+      "reward_std": 0.3170931892469525,
+      "rewards/accuracy_reward": 0.167578125,
+      "rewards/format_reward": 0.437109375,
       "step": 105
     },
     {
-      "completion_length": 527.8218933105469,
+      "completion_length": 403.58828125,
       "epoch": 0.38869257950530034,
-      "grad_norm": 0.34282830357551575,
-      "kl": 0.4187744140625,
+      "grad_norm": 0.534578040818149,
+      "kl": 1.238232421875,
       "learning_rate": 1.5387476295779737e-05,
-      "loss": 0.0168,
-      "reward": 0.11015625298023224,
-      "reward_std": 0.1587713214568794,
-      "rewards/accuracy_reward": 0.02656250069849193,
-      "rewards/format_reward": 0.08359375258442014,
+      "loss": 0.0495,
+      "reward": 0.597265625,
+      "reward_std": 0.31322619933635,
+      "rewards/accuracy_reward": 0.1671875,
+      "rewards/format_reward": 0.430078125,
       "step": 110
     },
     {
-      "completion_length": 535.7112144470215,
+      "completion_length": 408.934375,
       "epoch": 0.40636042402826855,
-      "grad_norm": 0.5226835012435913,
-      "kl": 0.46435546875,
+      "grad_norm": 0.41472109596238,
+      "kl": 1.536083984375,
       "learning_rate": 1.4856507733875837e-05,
-      "loss": 0.0186,
-      "reward": 0.10234375311993063,
-      "reward_std": 0.1523356933146715,
-      "rewards/accuracy_reward": 0.024479167209938167,
-      "rewards/format_reward": 0.07786458530463278,
+      "loss": 0.0614,
+      "reward": 0.6015625,
+      "reward_std": 0.3082543543539941,
+      "rewards/accuracy_reward": 0.173828125,
+      "rewards/format_reward": 0.427734375,
       "step": 115
     },
     {
-      "completion_length": 531.3724109649659,
+      "completion_length": 393.684375,
       "epoch": 0.42402826855123676,
-      "grad_norm": 0.3377923369407654,
-      "kl": 0.4623779296875,
+      "grad_norm": 1.0671198220931404,
+      "kl": 1.219482421875,
       "learning_rate": 1.4306971477188223e-05,
-      "loss": 0.0185,
-      "reward": 0.11875000363215804,
-      "reward_std": 0.1620171807706356,
-      "rewards/accuracy_reward": 0.03020833395421505,
-      "rewards/format_reward": 0.08854166890960187,
+      "loss": 0.0488,
+      "reward": 0.602734375,
+      "reward_std": 0.31101649152114985,
+      "rewards/accuracy_reward": 0.15546875,
+      "rewards/format_reward": 0.447265625,
       "step": 120
     },
     {
-      "completion_length": 543.5169471740722,
+      "completion_length": 411.417578125,
       "epoch": 0.4416961130742049,
-      "grad_norm": 0.4957655370235443,
-      "kl": 0.448388671875,
+      "grad_norm": 0.46043384834399087,
+      "kl": 1.910693359375,
       "learning_rate": 1.3740968546047935e-05,
-      "loss": 0.0179,
-      "reward": 0.09843750277068467,
-      "reward_std": 0.1416312349960208,
-      "rewards/accuracy_reward": 0.022656250558793545,
-      "rewards/format_reward": 0.07578125186264514,
+      "loss": 0.0764,
+      "reward": 0.60390625,
+      "reward_std": 0.29941552053205667,
+      "rewards/accuracy_reward": 0.16328125,
+      "rewards/format_reward": 0.440625,
       "step": 125
     },
     {
-      "completion_length": 522.5328289031983,
+      "completion_length": 397.962890625,
       "epoch": 0.45936395759717313,
-      "grad_norm": 0.49198487401008606,
-      "kl": 0.453369140625,
+      "grad_norm": 0.6111314907455481,
+      "kl": 0.9618408203125,
       "learning_rate": 1.3160662917174045e-05,
-      "loss": 0.0181,
-      "reward": 0.10729166988749057,
-      "reward_std": 0.15672538382932544,
-      "rewards/accuracy_reward": 0.024479167279787363,
-      "rewards/format_reward": 0.08281250211875886,
+      "loss": 0.0385,
+      "reward": 0.585546875,
+      "reward_std": 0.31212134528905155,
+      "rewards/accuracy_reward": 0.14765625,
+      "rewards/format_reward": 0.437890625,
       "step": 130
     },
     {
-      "completion_length": 534.0132961273193,
+      "completion_length": 401.703515625,
       "epoch": 0.47703180212014135,
-      "grad_norm": 0.4501975178718567,
-      "kl": 0.90474853515625,
+      "grad_norm": 1.4967183682261056,
+      "kl": 1.99873046875,
       "learning_rate": 1.2568273250226681e-05,
-      "loss": 0.0361,
-      "reward": 0.11223958637565375,
-      "reward_std": 0.1586504613980651,
-      "rewards/accuracy_reward": 0.032031250884756446,
-      "rewards/format_reward": 0.08020833532791585,
+      "loss": 0.08,
+      "reward": 0.61171875,
+      "reward_std": 0.2927863945718855,
+      "rewards/accuracy_reward": 0.1671875,
+      "rewards/format_reward": 0.44453125,
       "step": 135
     },
     {
-      "completion_length": 538.9510597229004,
+      "completion_length": 406.8,
       "epoch": 0.49469964664310956,
-      "grad_norm": 0.8263593316078186,
-      "kl": 0.4780517578125,
+      "grad_norm": 0.917377230805497,
+      "kl": 1.26953125,
       "learning_rate": 1.1966064405292887e-05,
-      "loss": 0.0191,
-      "reward": 0.10807291991077364,
-      "reward_std": 0.15537221673876048,
-      "rewards/accuracy_reward": 0.02109375048894435,
-      "rewards/format_reward": 0.08697916909586638,
+      "loss": 0.0508,
+      "reward": 0.59765625,
+      "reward_std": 0.31377862663939593,
+      "rewards/accuracy_reward": 0.168359375,
+      "rewards/format_reward": 0.429296875,
       "step": 140
     },
     {
-      "completion_length": 535.6718933105469,
+      "completion_length": 391.3640625,
       "epoch": 0.5123674911660777,
-      "grad_norm": 0.37266990542411804,
-      "kl": 0.430517578125,
+      "grad_norm": 0.5259506143043062,
+      "kl": 1.326171875,
       "learning_rate": 1.1356338783736256e-05,
-      "loss": 0.0172,
-      "reward": 0.10286458616610616,
-      "reward_std": 0.15098252883180976,
-      "rewards/accuracy_reward": 0.024218750512227417,
-      "rewards/format_reward": 0.07864583553746343,
+      "loss": 0.053,
+      "reward": 0.6125,
+      "reward_std": 0.28615726907737554,
+      "rewards/accuracy_reward": 0.166015625,
+      "rewards/format_reward": 0.446484375,
       "step": 145
     },
     {
-      "completion_length": 535.9765785217285,
+      "completion_length": 397.0296875,
       "epoch": 0.5300353356890459,
-      "grad_norm": 0.32091596722602844,
-      "kl": 0.41318359375,
+      "grad_norm": 0.6823442998587644,
+      "kl": 1.138134765625,
       "learning_rate": 1.0741427525516463e-05,
-      "loss": 0.0165,
-      "reward": 0.10651041977107525,
-      "reward_std": 0.15906913336366416,
-      "rewards/accuracy_reward": 0.027083333930931985,
-      "rewards/format_reward": 0.07942708558402956,
+      "loss": 0.0455,
+      "reward": 0.6140625,
+      "reward_std": 0.2828427059110254,
+      "rewards/accuracy_reward": 0.168359375,
+      "rewards/format_reward": 0.445703125,
       "step": 150
     },
     {
-      "completion_length": 536.0502750396729,
+      "completion_length": 407.55390625,
       "epoch": 0.5477031802120141,
-      "grad_norm": 10.207870483398438,
-      "kl": 0.556298828125,
+      "grad_norm": 5.1945886494720215,
+      "kl": 1.53837890625,
       "learning_rate": 1.012368159663363e-05,
-      "loss": 0.0222,
-      "reward": 0.10286458663176745,
-      "reward_std": 0.14541662586852908,
-      "rewards/accuracy_reward": 0.02213541720993817,
-      "rewards/format_reward": 0.08072916818782687,
+      "loss": 0.0615,
+      "reward": 0.57109375,
+      "reward_std": 0.3038349375128746,
+      "rewards/accuracy_reward": 0.14375,
+      "rewards/format_reward": 0.42734375,
       "step": 155
     },
     {
-      "completion_length": 534.5656383514404,
+      "completion_length": 401.918359375,
       "epoch": 0.5653710247349824,
-      "grad_norm": 0.3280162513256073,
-      "kl": 0.4006103515625,
+      "grad_norm": 0.4307285006305928,
+      "kl": 1.49169921875,
       "learning_rate": 9.505462800772612e-06,
-      "loss": 0.016,
-      "reward": 0.11484375381842256,
-      "reward_std": 0.1651745643466711,
-      "rewards/accuracy_reward": 0.026041667303070427,
-      "rewards/format_reward": 0.08880208584014326,
+      "loss": 0.0597,
+      "reward": 0.60546875,
+      "reward_std": 0.3049397937953472,
+      "rewards/accuracy_reward": 0.1625,
+      "rewards/format_reward": 0.44296875,
       "step": 160
     },
     {
-      "completion_length": 540.3789237976074,
+      "completion_length": 402.686328125,
       "epoch": 0.5830388692579506,
-      "grad_norm": 3.008737325668335,
-      "kl": 0.543115234375,
+      "grad_norm": 0.49030947447366735,
+      "kl": 1.36279296875,
       "learning_rate": 8.889134749511956e-06,
-      "loss": 0.0217,
-      "reward": 0.1114583365386352,
-      "reward_std": 0.16135679064318537,
-      "rewards/accuracy_reward": 0.025260417233221234,
-      "rewards/format_reward": 0.08619791895616799,
+      "loss": 0.0545,
+      "reward": 0.590625,
+      "reward_std": 0.3038349383510649,
+      "rewards/accuracy_reward": 0.153125,
+      "rewards/format_reward": 0.4375,
       "step": 165
     },
     {
-      "completion_length": 537.1291847229004,
+      "completion_length": 405.9875,
       "epoch": 0.6007067137809188,
-      "grad_norm": 0.39234212040901184,
-      "kl": 0.503076171875,
+      "grad_norm": 0.4352892915737104,
+      "kl": 1.3828125,
       "learning_rate": 8.277053825620836e-06,
-      "loss": 0.0201,
-      "reward": 0.11015625316649676,
-      "reward_std": 0.14803447835147382,
-      "rewards/accuracy_reward": 0.02421875048894435,
-      "rewards/format_reward": 0.08593750237487256,
+      "loss": 0.0553,
+      "reward": 0.61328125,
+      "reward_std": 0.3027300829067826,
+      "rewards/accuracy_reward": 0.1546875,
+      "rewards/format_reward": 0.45859375,
       "step": 170
     },
     {
-      "completion_length": 539.1065277099609,
+      "completion_length": 401.63359375,
       "epoch": 0.6183745583038869,
-      "grad_norm": 0.4099801182746887,
-      "kl": 0.456982421875,
+      "grad_norm": 0.8900367662135797,
+      "kl": 1.16708984375,
       "learning_rate": 7.671560173993588e-06,
-      "loss": 0.0183,
-      "reward": 0.1072916699340567,
-      "reward_std": 0.15606499183923006,
-      "rewards/accuracy_reward": 0.026822917303070425,
-      "rewards/format_reward": 0.08046875232830644,
+      "loss": 0.0467,
+      "reward": 0.615625,
+      "reward_std": 0.3038349374197423,
+      "rewards/accuracy_reward": 0.18046875,
+      "rewards/format_reward": 0.43515625,
       "step": 175
     },
     {
-      "completion_length": 531.5690284729004,
+      "completion_length": 399.348828125,
       "epoch": 0.6360424028268551,
-      "grad_norm": 0.596886396408081,
-      "kl": 0.5114990234375,
+      "grad_norm": 0.8123164664250848,
+      "kl": 1.635009765625,
       "learning_rate": 7.07496875466589e-06,
-      "loss": 0.0205,
-      "reward": 0.1161458363989368,
-      "reward_std": 0.16755069950595497,
-      "rewards/accuracy_reward": 0.028125000651925802,
-      "rewards/format_reward": 0.08802083584014327,
+      "loss": 0.0654,
+      "reward": 0.61640625,
+      "reward_std": 0.29720581048168243,
+      "rewards/accuracy_reward": 0.1640625,
+      "rewards/format_reward": 0.45234375,
       "step": 180
     },
     {
-      "completion_length": 535.6968925476074,
+      "completion_length": 401.88515625,
       "epoch": 0.6537102473498233,
-      "grad_norm": 0.31039756536483765,
-      "kl": 0.4974609375,
+      "grad_norm": 0.5468048622490435,
+      "kl": 1.2939208984375,
       "learning_rate": 6.489560492119225e-06,
-      "loss": 0.0199,
-      "reward": 0.10598958644550294,
-      "reward_std": 0.15061994921416044,
-      "rewards/accuracy_reward": 0.021875000512227415,
-      "rewards/format_reward": 0.08411458577029407,
+      "loss": 0.0518,
+      "reward": 0.607421875,
+      "reward_std": 0.3054922170005739,
+      "rewards/accuracy_reward": 0.15546875,
+      "rewards/format_reward": 0.451953125,
       "step": 185
     },
     {
-      "completion_length": 540.5557464599609,
+      "completion_length": 397.83359375,
       "epoch": 0.6713780918727915,
-      "grad_norm": 1.3244882822036743,
-      "kl": 0.4893798828125,
+      "grad_norm": 0.7244711255317006,
+      "kl": 1.36591796875,
       "learning_rate": 5.9175735547120975e-06,
-      "loss": 0.0196,
-      "reward": 0.1151041706558317,
-      "reward_std": 0.15738577228039502,
-      "rewards/accuracy_reward": 0.029947917396202683,
-      "rewards/format_reward": 0.08515625260770321,
+      "loss": 0.0546,
+      "reward": 0.6109375,
+      "reward_std": 0.3181980476714671,
+      "rewards/accuracy_reward": 0.155859375,
+      "rewards/format_reward": 0.455078125,
       "step": 190
     },
     {
-      "completion_length": 524.1333465576172,
+      "completion_length": 413.570703125,
       "epoch": 0.6890459363957597,
-      "grad_norm": 1.8887150287628174,
-      "kl": 0.474072265625,
+      "grad_norm": 0.42585359101119413,
+      "kl": 1.764208984375,
       "learning_rate": 5.361194797579108e-06,
-      "loss": 0.019,
-      "reward": 0.1049479192122817,
-      "reward_std": 0.14905744856223463,
-      "rewards/accuracy_reward": 0.02500000058207661,
-      "rewards/format_reward": 0.07994791893288493,
+      "loss": 0.0706,
+      "reward": 0.6046875,
+      "reward_std": 0.3093592093326151,
+      "rewards/accuracy_reward": 0.1671875,
+      "rewards/format_reward": 0.4375,
       "step": 195
     },
     {
-      "completion_length": 542.0661624908447,
+      "completion_length": 399.438671875,
       "epoch": 0.7067137809187279,
-      "grad_norm": 0.528878390789032,
-      "kl": 0.551904296875,
+      "grad_norm": 0.47550986420528907,
+      "kl": 1.2271484375,
       "learning_rate": 4.8225514017138205e-06,
-      "loss": 0.0221,
-      "reward": 0.1114583361428231,
-      "reward_std": 0.15434924876317382,
-      "rewards/accuracy_reward": 0.024739583930931987,
-      "rewards/format_reward": 0.08671875244472176,
+      "loss": 0.0491,
+      "reward": 0.60546875,
+      "reward_std": 0.3038349355570972,
+      "rewards/accuracy_reward": 0.1578125,
+      "rewards/format_reward": 0.44765625,
       "step": 200
     },
     {
       "epoch": 0.7067137809187279,
-      "eval_completion_length": 569.0494995117188,
-      "eval_kl": 0.56201171875,
-      "eval_loss": 0.022066345438361168,
-      "eval_reward": 0.1171875037252903,
-      "eval_reward_std": 0.18042195588350296,
-      "eval_rewards/accuracy_reward": 0.026041666977107525,
-      "eval_rewards/format_reward": 0.0911458358168602,
-      "eval_runtime": 71.3459,
-      "eval_samples_per_second": 1.388,
-      "eval_steps_per_second": 0.028,
+      "eval_completion_length": 383.06652069091797,
+      "eval_kl": 1.0625,
+      "eval_loss": 0.043225545436143875,
+      "eval_reward": 0.6328125,
+      "eval_reward_std": 0.2651650458574295,
+      "eval_rewards/accuracy_reward": 0.16015625,
+      "eval_rewards/format_reward": 0.47265625,
+      "eval_runtime": 64.1406,
+      "eval_samples_per_second": 1.543,
+      "eval_steps_per_second": 0.031,
       "step": 200
     },
     {
-      "completion_length": 534.8948085784912,
+      "completion_length": 410.916796875,
       "epoch": 0.7243816254416962,
-      "grad_norm": 0.43772202730178833,
-      "kl": 0.576123046875,
+      "grad_norm": 0.4152907308781029,
+      "kl": 1.306787109375,
       "learning_rate": 4.303702741201431e-06,
-      "loss": 0.023,
-      "reward": 0.10651041900273413,
-      "reward_std": 0.1511918638832867,
-      "rewards/accuracy_reward": 0.025260417209938168,
-      "rewards/format_reward": 0.08125000228174031,
+      "loss": 0.0523,
+      "reward": 0.605859375,
+      "reward_std": 0.29554852955043315,
+      "rewards/accuracy_reward": 0.162890625,
+      "rewards/format_reward": 0.44296875,
       "step": 205
     },
     {
-      "completion_length": 533.3744941711426,
+      "completion_length": 420.941796875,
       "epoch": 0.7420494699646644,
-      "grad_norm": 0.927462100982666,
-      "kl": 0.59678955078125,
+      "grad_norm": 0.3864466357905039,
+      "kl": 1.347509765625,
       "learning_rate": 3.8066325096949153e-06,
-      "loss": 0.0239,
-      "reward": 0.11250000314321369,
-      "reward_std": 0.161026596929878,
-      "rewards/accuracy_reward": 0.02656250053551048,
-      "rewards/format_reward": 0.08593750209547579,
+      "loss": 0.0539,
+      "reward": 0.594140625,
+      "reward_std": 0.3010728007182479,
+      "rewards/accuracy_reward": 0.16484375,
+      "rewards/format_reward": 0.429296875,
       "step": 210
     },
     {
-      "completion_length": 536.620327758789,
+      "completion_length": 405.008984375,
       "epoch": 0.7597173144876325,
-      "grad_norm": 0.38544711470603943,
-      "kl": 0.5203857421875,
+      "grad_norm": 0.4071374966900544,
+      "kl": 1.3837890625,
       "learning_rate": 3.3332411362372063e-06,
-      "loss": 0.0208,
-      "reward": 0.10703125295694918,
-      "reward_std": 0.1591015163809061,
-      "rewards/accuracy_reward": 0.023437500605359672,
-      "rewards/format_reward": 0.0835937523515895,
+      "loss": 0.0554,
+      "reward": 0.5796875,
+      "reward_std": 0.31156891863793135,
+      "rewards/accuracy_reward": 0.155859375,
+      "rewards/format_reward": 0.423828125,
       "step": 215
     },
     {
-      "completion_length": 534.7442867279053,
+      "completion_length": 400.690234375,
       "epoch": 0.7773851590106007,
-      "grad_norm": 0.37316420674324036,
-      "kl": 0.4793701171875,
+      "grad_norm": 0.48756085064617083,
+      "kl": 1.277734375,
       "learning_rate": 2.8853385194256677e-06,
-      "loss": 0.0192,
-      "reward": 0.10937500353902578,
-      "reward_std": 0.16291929073631764,
-      "rewards/accuracy_reward": 0.025000000605359674,
-      "rewards/format_reward": 0.08437500270083546,
+      "loss": 0.0511,
+      "reward": 0.6046875,
+      "reward_std": 0.31488348012790085,
+      "rewards/accuracy_reward": 0.173046875,
+      "rewards/format_reward": 0.431640625,
       "step": 220
     },
     {
-      "completion_length": 527.4945472717285,
+      "completion_length": 418.2671875,
       "epoch": 0.7950530035335689,
-      "grad_norm": 0.4162587821483612,
-      "kl": 0.5058837890625,
+      "grad_norm": 0.37777218952387104,
+      "kl": 1.430517578125,
       "learning_rate": 2.464637107698046e-06,
-      "loss": 0.0202,
-      "reward": 0.10416667007375509,
-      "reward_std": 0.1492991691455245,
-      "rewards/accuracy_reward": 0.024739583814516665,
-      "rewards/format_reward": 0.07942708535119891,
+      "loss": 0.0573,
+      "reward": 0.59375,
+      "reward_std": 0.312673770962283,
+      "rewards/accuracy_reward": 0.169140625,
+      "rewards/format_reward": 0.424609375,
       "step": 225
     },
     {
-      "completion_length": 532.8752773284912,
+      "completion_length": 409.259375,
       "epoch": 0.8127208480565371,
-      "grad_norm": 0.40629276633262634,
-      "kl": 0.521826171875,
+      "grad_norm": 0.5380959008484876,
+      "kl": 1.55126953125,
       "learning_rate": 2.072745352195794e-06,
-      "loss": 0.0209,
-      "reward": 0.10807291981764137,
-      "reward_std": 0.15311694331467152,
-      "rewards/accuracy_reward": 0.02213541716337204,
-      "rewards/format_reward": 0.08593750256113708,
+      "loss": 0.0621,
+      "reward": 0.5984375,
+      "reward_std": 0.2938912484794855,
+      "rewards/accuracy_reward": 0.17734375,
+      "rewards/format_reward": 0.42109375,
       "step": 230
     },
     {
-      "completion_length": 528.4742362976074,
+      "completion_length": 408.725,
       "epoch": 0.8303886925795053,
-      "grad_norm": 0.3422054350376129,
-      "kl": 0.562890625,
+      "grad_norm": 0.6087277900031041,
+      "kl": 1.276611328125,
       "learning_rate": 1.7111615572361628e-06,
-      "loss": 0.0225,
-      "reward": 0.10833333611954003,
-      "reward_std": 0.15480030244216322,
-      "rewards/accuracy_reward": 0.027343750465661287,
-      "rewards/format_reward": 0.08098958558402955,
+      "loss": 0.0511,
+      "reward": 0.608203125,
+      "reward_std": 0.32096018167212603,
+      "rewards/accuracy_reward": 0.166015625,
+      "rewards/format_reward": 0.4421875,
       "step": 235
     },
     {
-      "completion_length": 534.3093910217285,
+      "completion_length": 408.4484375,
       "epoch": 0.8480565371024735,
-      "grad_norm": 0.28931647539138794,
-      "kl": 0.5560546875,
+      "grad_norm": 0.40330618192163126,
+      "kl": 1.2666015625,
       "learning_rate": 1.381268151904298e-06,
-      "loss": 0.0222,
-      "reward": 0.10286458667833358,
-      "reward_std": 0.15053147487342358,
-      "rewards/accuracy_reward": 0.023437500605359672,
-      "rewards/format_reward": 0.07942708572372795,
+      "loss": 0.0507,
+      "reward": 0.591015625,
+      "reward_std": 0.2999679483473301,
+      "rewards/accuracy_reward": 0.1625,
+      "rewards/format_reward": 0.428515625,
       "step": 240
     },
     {
-      "completion_length": 544.1958484649658,
+      "completion_length": 411.65390625,
       "epoch": 0.8657243816254417,
-      "grad_norm": 0.5518237352371216,
-      "kl": 0.5116455078125,
+      "grad_norm": 0.5432070185726592,
+      "kl": 1.262109375,
       "learning_rate": 1.0843264046665558e-06,
-      "loss": 0.0205,
-      "reward": 0.10651042016688735,
-      "reward_std": 0.15471182586625218,
-      "rewards/accuracy_reward": 0.02213541720993817,
-      "rewards/format_reward": 0.08437500256113709,
+      "loss": 0.0505,
+      "reward": 0.5953125,
+      "reward_std": 0.3016252295579761,
+      "rewards/accuracy_reward": 0.15703125,
+      "rewards/format_reward": 0.43828125,
       "step": 245
     },
     {
-      "completion_length": 538.685956954956,
+      "completion_length": 409.023046875,
       "epoch": 0.8833922261484098,
-      "grad_norm": 0.4649152159690857,
-      "kl": 0.5036376953125,
+      "grad_norm": 0.3703261485488551,
+      "kl": 1.469970703125,
       "learning_rate": 8.214716012124491e-07,
-      "loss": 0.0201,
-      "reward": 0.1078125033993274,
-      "reward_std": 0.15783682689070702,
-      "rewards/accuracy_reward": 0.027083333837799727,
-      "rewards/format_reward": 0.0807291688863188,
+      "loss": 0.0588,
+      "reward": 0.605078125,
+      "reward_std": 0.3021776580251753,
+      "rewards/accuracy_reward": 0.16484375,
+      "rewards/format_reward": 0.440234375,
       "step": 250
     },
     {
-      "completion_length": 534.5523597717286,
+      "completion_length": 407.982421875,
       "epoch": 0.901060070671378,
-      "grad_norm": 0.37897783517837524,
-      "kl": 0.5443115234375,
+      "grad_norm": 0.48718776424035376,
+      "kl": 1.395263671875,
       "learning_rate": 5.937087039615619e-07,
-      "loss": 0.0218,
-      "reward": 0.10833333663176745,
-      "reward_std": 0.15188463851809503,
-      "rewards/accuracy_reward": 0.031770834187045696,
-      "rewards/format_reward": 0.07656250230502337,
+      "loss": 0.0558,
+      "reward": 0.61328125,
+      "reward_std": 0.2861572677269578,
+      "rewards/accuracy_reward": 0.171875,
+      "rewards/format_reward": 0.44140625,
       "step": 255
     },
     {
-      "completion_length": 534.8096504211426,
+      "completion_length": 399.203125,
       "epoch": 0.9187279151943463,
-      "grad_norm": 0.3674253523349762,
-      "kl": 0.5176025390625,
+      "grad_norm": 0.4674025132985681,
+      "kl": 1.384521484375,
       "learning_rate": 4.019085098303077e-07,
-      "loss": 0.0207,
-      "reward": 0.11614583698101341,
-      "reward_std": 0.16189632220193745,
-      "rewards/accuracy_reward": 0.033072917419485745,
-      "rewards/format_reward": 0.08307291870005429,
+      "loss": 0.0554,
+      "reward": 0.599609375,
+      "reward_std": 0.30438736486248674,
+      "rewards/accuracy_reward": 0.154296875,
+      "rewards/format_reward": 0.4453125,
       "step": 260
     },
     {
-      "completion_length": 536.2896011352539,
+      "completion_length": 407.852734375,
       "epoch": 0.9363957597173145,
-      "grad_norm": 0.42442551255226135,
-      "kl": 0.5372802734375,
+      "grad_norm": 0.518993833517968,
+      "kl": 1.352685546875,
       "learning_rate": 2.4680432094837394e-07,
-      "loss": 0.0215,
-      "reward": 0.11093750363215804,
-      "reward_std": 0.1559441326186061,
-      "rewards/accuracy_reward": 0.02708333395421505,
-      "rewards/format_reward": 0.0838541690260172,
+      "loss": 0.0541,
+      "reward": 0.5921875,
+      "reward_std": 0.2861572689376771,
+      "rewards/accuracy_reward": 0.1609375,
+      "rewards/format_reward": 0.43125,
       "step": 265
     },
     {
-      "completion_length": 542.7255397796631,
+      "completion_length": 418.2390625,
       "epoch": 0.9540636042402827,
-      "grad_norm": 0.4530044496059418,
-      "kl": 0.5424072265625,
+      "grad_norm": 0.41910418773784364,
+      "kl": 1.345751953125,
       "learning_rate": 1.289891410535593e-07,
-      "loss": 0.0217,
-      "reward": 0.09973958623595536,
-      "reward_std": 0.14105932042002678,
-      "rewards/accuracy_reward": 0.021614583884365858,
-      "rewards/format_reward": 0.07812500209547579,
+      "loss": 0.0538,
+      "reward": 0.598828125,
+      "reward_std": 0.3209601787850261,
+      "rewards/accuracy_reward": 0.168359375,
+      "rewards/format_reward": 0.43046875,
       "step": 270
     },
     {
-      "completion_length": 539.1974124908447,
+      "completion_length": 403.1046875,
       "epoch": 0.9717314487632509,
-      "grad_norm": 0.3238506019115448,
-      "kl": 0.5445068359375,
+      "grad_norm": 0.5629306163631225,
+      "kl": 1.38427734375,
       "learning_rate": 4.8913408283934874e-08,
-      "loss": 0.0218,
-      "reward": 0.09322916918899864,
-      "reward_std": 0.13790193693712355,
-      "rewards/accuracy_reward": 0.024479167140088975,
-      "rewards/format_reward": 0.0687500016996637,
+      "loss": 0.0554,
+      "reward": 0.587109375,
+      "reward_std": 0.31101649068295956,
+      "rewards/accuracy_reward": 0.155078125,
+      "rewards/format_reward": 0.43203125,
       "step": 275
     },
     {
-      "completion_length": 545.6544456481934,
+      "completion_length": 396.14375,
       "epoch": 0.9893992932862191,
-      "grad_norm": 7.467169284820557,
-      "kl": 0.6452880859375,
+      "grad_norm": 0.40077217392258757,
+      "kl": 1.292138671875,
       "learning_rate": 6.883273035447335e-09,
-      "loss": 0.0258,
-      "reward": 0.10755208674818277,
-      "reward_std": 0.15865046214312314,
-      "rewards/accuracy_reward": 0.02864583404734731,
-      "rewards/format_reward": 0.07890625216532499,
+      "loss": 0.0517,
+      "reward": 0.61484375,
+      "reward_std": 0.29610095573589207,
+      "rewards/accuracy_reward": 0.16953125,
+      "rewards/format_reward": 0.4453125,
       "step": 280
     },
     {
-      "completion_length": 522.6118488311768,
+      "completion_length": 388.8455181121826,
       "epoch": 1.0,
-      "kl": 0.5454915364583334,
-      "reward": 0.10503472515847534,
-      "reward_std": 0.1531572287591795,
-      "rewards/accuracy_reward": 0.026475695117066305,
-      "rewards/format_reward": 0.07855903015782435,
+      "kl": 1.29052734375,
+      "reward": 0.6243489583333334,
+      "reward_std": 0.3176456190024813,
+      "rewards/accuracy_reward": 0.17838541666666666,
+      "rewards/format_reward": 0.4459635416666667,
       "step": 283,
       "total_flos": 0.0,
-      "train_loss": 0.1736428747392735,
-      "train_runtime": 54182.9853,
-      "train_samples_per_second": 1.337,
-      "train_steps_per_second": 0.005
+      "train_loss": 27.009561450669516,
+      "train_runtime": 47693.8258,
+      "train_samples_per_second": 1.519,
+      "train_steps_per_second": 0.006
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3223c81b4d5af21a4323b99f7322808c9598dd9bb65b24cbea2cf188582b0bc5
+oid sha256:a571ea342950b818515612b20599050de613be01d10ca84f6715a795c3f31929
 size 7544