| data: | |
| tokenizer: null | |
| train_files: numina_ds_train_sample.parquet | |
| val_files: matheval.parquet | |
| prompt_key: prompt | |
| max_prompt_length: 1024 | |
| max_response_length: 2048 | |
| train_batch_size: 32 | |
| val_batch_size: 640 | |
| return_raw_input_ids: false | |
| return_raw_chat: false | |
| shuffle: true | |
| apply_chat_template: true | |
| actor_rollout_ref: | |
| hybrid_engine: true | |
| model: | |
| path: Qwen/Qwen2.5-Math-7B | |
| revision: main | |
| external_lib: null | |
| override_config: {} | |
| enable_gradient_checkpointing: true | |
| use_remove_padding: false | |
| save_hf_repo_id: RyanYr/brm-numina-qwen2.5math-7B-base-lr5e-7constant-n4 | |
| actor: | |
| loss: brm | |
| brm: | |
| norm_factor: value | |
| qlearn: | |
| use_replaybuffer: true | |
| replaybuffer_size: 32 | |
| replaybuffer_sample_size: 32 | |
| replaybuffer_update_size: 32 | |
| replaybuffer_update_reward_threshold: 1.0 | |
| strategy: fsdp | |
| ppo_mini_batch_size: 64 | |
| ppo_micro_batch_size: null | |
| ppo_micro_batch_size_per_gpu: 4 | |
| use_dynamic_bsz: false | |
| ppo_max_token_len_per_gpu: 16384 | |
| grad_clip: 1.0 | |
| clip_ratio: 0.2 | |
| entropy_coeff: 0.001 | |
| use_kl_loss: false | |
| kl_loss_coef: 0.001 | |
| kl_loss_type: low_var_kl | |
| ppo_epochs: 1 | |
| shuffle: false | |
| ulysses_sequence_parallel_size: 1 | |
| optim: | |
| lr: 5.0e-07 | |
| lr_warmup_steps_ratio: 0 | |
| min_lr_ratio: null | |
| warmup_style: constant | |
| total_training_steps: 12496 | |
| fsdp_config: | |
| wrap_policy: | |
| min_num_params: 0 | |
| param_offload: false | |
| optimizer_offload: false | |
| fsdp_size: -1 | |
| ref: | |
| fsdp_config: | |
| param_offload: false | |
| wrap_policy: | |
| min_num_params: 0 | |
| log_prob_micro_batch_size: null | |
| log_prob_micro_batch_size_per_gpu: 4 | |
| log_prob_use_dynamic_bsz: false | |
| log_prob_max_token_len_per_gpu: 16384 | |
| ulysses_sequence_parallel_size: 1 | |
| rollout: | |
| name: vllm | |
| temperature: 1.0 | |
| top_k: -1 | |
| top_p: 1 | |
| prompt_length: 1024 | |
| response_length: 2048 | |
| dtype: bfloat16 | |
| gpu_memory_utilization: 0.6 | |
| ignore_eos: false | |
| enforce_eager: true | |
| free_cache_engine: true | |
| load_format: dummy_dtensor | |
| tensor_model_parallel_size: 4 | |
| max_num_batched_tokens: 8192 | |
| max_num_seqs: 1024 | |
| log_prob_micro_batch_size: null | |
| log_prob_micro_batch_size_per_gpu: 4 | |
| log_prob_use_dynamic_bsz: false | |
| log_prob_max_token_len_per_gpu: 16384 | |
| disable_log_stats: true | |
| enable_chunked_prefill: true | |
| do_sample: true | |
| 'n': 4 | |
| seed: 42 | |
| reward_model: | |
| reward_manager: prime | |
| extra_rwfn_argdict: | |
| format_score: 0 | |
| score: 1.0 | |
| algorithm: | |
| kl_penalty: kl | |
| kl_ctrl: | |
| type: fixed | |
| kl_coef: 0.1 | |
| trainer: | |
| total_epochs: 1 | |
| total_training_steps: null | |
| project_name: value-LLM | |
| experiment_name: brm-numina-qwen2.5math-7B-base_lr5e-7constant-n4 | |
| logger: | |
| - wandb | |
| val_generations_to_log_to_wandb: 0 | |
| nnodes: 1 | |
| n_gpus_per_node: 4 | |
| save_freq: 80 | |
| resume_mode: auto | |
| resume_from_path: false | |
| test_freq: 20 | |
| critic_warmup: 0 | |
| default_hdfs_dir: null | |
| remove_previous_ckpt_in_save: true | |
| del_local_ckpt_after_load: false | |
| default_local_dir: ./BRM | |
| hf_token: null | |
| resume_from_hf: | |
| enable: true | |
| hf_repo_id: RyanYr/brm-numina-qwen2.5math-7B-base-lr5e-7constant-n4 | |
| hf_token: null | |
| revision: 1932c649200c975a4b3ae08f1ed04561da3736ff | |
| val_before_train: true | |