data:
  tokenizer: null
  train_files: numina_ds_train_sample.parquet
  val_files: matheval.parquet
  prompt_key: prompt
  max_prompt_length: 1024
  max_response_length: 2048
  train_batch_size: 128
  val_batch_size: 640
  return_raw_input_ids: false
  return_raw_chat: false
  shuffle: true
  apply_chat_template: true
actor_rollout_ref:
  hybrid_engine: true
  model:
    path: Qwen/Qwen2.5-Math-7B
    external_lib: null
    override_config: {}
    enable_gradient_checkpointing: true
    use_remove_padding: false
    save_hf_repo_id: RyanYr/numina-qwen2.5math-7Bbase-ppo_actor
  actor:
    strategy: fsdp
    ppo_mini_batch_size: 64
    ppo_micro_batch_size: null
    ppo_micro_batch_size_per_gpu: 2
    use_dynamic_bsz: false
    ppo_max_token_len_per_gpu: 16384
    grad_clip: 1.0
    clip_ratio: 0.2
    entropy_coeff: 0.001
    use_kl_loss: false
    kl_loss_coef: 0.001
    kl_loss_type: low_var_kl
    ppo_epochs: 1
    shuffle: false
    ulysses_sequence_parallel_size: 1
    optim:
      lr: 1.0e-06
      lr_warmup_steps_ratio: 0.0
      min_lr_ratio: null
      warmup_style: constant
      total_training_steps: 3124
    fsdp_config:
      wrap_policy:
        min_num_params: 0
      param_offload: false
      optimizer_offload: false
      fsdp_size: -1
  ref:
    fsdp_config:
      param_offload: false
      wrap_policy:
        min_num_params: 0
    log_prob_micro_batch_size: null
    log_prob_micro_batch_size_per_gpu: 2
    log_prob_use_dynamic_bsz: false
    log_prob_max_token_len_per_gpu: 16384
    ulysses_sequence_parallel_size: 1
  rollout:
    name: vllm
    temperature: 1.0
    top_k: -1
    top_p: 1
    prompt_length: 1024
    response_length: 2048
    dtype: bfloat16
    gpu_memory_utilization: 0.2
    ignore_eos: false
    enforce_eager: true
    free_cache_engine: true
    load_format: dummy_dtensor
    tensor_model_parallel_size: 4
    max_num_batched_tokens: 8192
    max_num_seqs: 1024
    log_prob_micro_batch_size: null
    log_prob_micro_batch_size_per_gpu: 2
    log_prob_use_dynamic_bsz: false
    log_prob_max_token_len_per_gpu: 16384
    disable_log_stats: true
    enable_chunked_prefill: true
    do_sample: true
    'n': 1
    seed: 42
critic:
  strategy: fsdp
  optim:
    lr: 1.0e-05
    lr_warmup_steps_ratio: 0.0
    min_lr_ratio: null
    warmup_style: constant
    total_training_steps: 3124
  model:
    path: Qwen/Qwen2.5-Math-7B
    tokenizer_path: Qwen/Qwen2.5-Math-7B
    override_config: {}
    external_lib: null
    enable_gradient_checkpointing: true
    use_remove_padding: false
    fsdp_config:
      param_offload: false
      optimizer_offload: false
      wrap_policy:
        min_num_params: 0
      fsdp_size: -1
    save_hf_repo_id: RyanYr/numina-qwen2.5math-7Bbase-ppo_critic
  ppo_mini_batch_size: 64
  ppo_micro_batch_size: 4
  ppo_micro_batch_size_per_gpu: null
  forward_micro_batch_size: 4
  forward_micro_batch_size_per_gpu: null
  use_dynamic_bsz: false
  ppo_max_token_len_per_gpu: 32768
  forward_max_token_len_per_gpu: 32768
  ulysses_sequence_parallel_size: 1
  ppo_epochs: 1
  shuffle: false
  grad_clip: 1.0
  cliprange_value: 0.5
reward_model:
  enable: false
  strategy: fsdp
  model:
    input_tokenizer: Qwen/Qwen2.5-Math-7B
    path: ~/models/FsfairX-LLaMA3-RM-v0.1
    external_lib: null
    use_remove_padding: false
    fsdp_config:
      min_num_params: 0
      param_offload: false
      fsdp_size: -1
  micro_batch_size: null
  micro_batch_size_per_gpu: null
  max_length: null
  ulysses_sequence_parallel_size: 1
  use_dynamic_bsz: false
  forward_max_token_len_per_gpu: 32768
  reward_manager: prime
algorithm:
  gamma: 1.0
  lam: 1.0
  adv_estimator: gae
  kl_penalty: kl
  kl_ctrl:
    type: fixed
    kl_coef: 0.001
trainer:
  total_epochs: 1
  total_training_steps: null
  project_name: value-LLM
  experiment_name: ppo-numina-qwen2.5math-7Bbase
  logger:
  - wandb
  val_generations_to_log_to_wandb: 0
  nnodes: 1
  n_gpus_per_node: 4
  save_freq: 100
  resume_mode: auto
  resume_from_path: false
  test_freq: 20
  critic_warmup: 0
  default_hdfs_dir: null
  remove_previous_ckpt_in_save: true
  del_local_ckpt_after_load: false
  default_local_dir: ./PPO
  hf_token: null
  resume_from_hf:
    enable: true
    actor_hf_repo_id: RyanYr/numina-qwen2.5math-7Bbase-ppo_actor
    actor_revision: aff7a826e225ddc044daec6d1f93f57e38b4f15b
    critic_hf_repo_id: RyanYr/numina-qwen2.5math-7Bbase-ppo_critic
    critic_revision: b98e588ce4c6c681ad584af7ef7506305647b205
    hf_token: null
  val_before_train: true