| prefix: /home/arqa39/.torchtune | |
| output_dir: /home/arqa39/.torchtune/fed_ppo/mistral_7b | |
| tokenizer: | |
| _component_: torchtune.models.mistral.mistral_tokenizer | |
| path: ${prefix}/models/Mistral-7B-Instruct-v0.2/tokenizer.model | |
| max_seq_len: null | |
| dataset: | |
| _component_: torchtune.datasets.text_completion_dataset | |
| source: trl-internal-testing/sentiment-trl-style | |
| split: train | |
| column: prompt | |
| add_eos: false | |
| policy: | |
| _component_: torchtune.models.mistral.lora_mistral_7b | |
| lora_attn_modules: | |
| - q_proj | |
| - k_proj | |
| - v_proj | |
| - output_proj | |
| apply_lora_to_mlp: true | |
| apply_lora_to_output: false | |
| lora_rank: 64 | |
| lora_alpha: 16 | |
| lora_dropout: 0.0 | |
| quantize_base: false | |
| valmod: | |
| _component_: torchtune.models.mistral._component_builders.lora_mistral_classifier | |
| attn_dropout: 0.0 | |
| embed_dim: 4096 | |
| intermediate_dim: 14336 | |
| max_seq_len: 32768 | |
| norm_eps: 1.0e-05 | |
| num_classes: 1 | |
| num_heads: 32 | |
| num_kv_heads: 8 | |
| num_layers: 32 | |
| vocab_size: 32001 | |
| lora_attn_modules: | |
| - q_proj | |
| - k_proj | |
| - v_proj | |
| - output_proj | |
| apply_lora_to_mlp: true | |
| apply_lora_to_output: = True | |
| lora_rank: 16 | |
| lora_alpha: 32 | |
| lora_dropout: 0.0 | |
| quantize_base: false | |
| checkpointer: | |
| _component_: torchtune.training.FullModelHFCheckpointer | |
| checkpoint_dir: ${prefix}/models/Mistral-7B-Instruct-v0.2/ | |
| checkpoint_files: | |
| - pytorch_model-00001-of-00003.bin | |
| - pytorch_model-00002-of-00003.bin | |
| - pytorch_model-00003-of-00003.bin | |
| recipe_checkpoint: null | |
| output_dir: ${output_dir}/policy | |
| model_type: MISTRAL | |
| value_checkpointer: | |
| _component_: torchtune.training.FullModelHFCheckpointer | |
| checkpoint_dir: ${prefix}/models/RM-Mistral-7B/ | |
| checkpoint_files: | |
| - model-00001-of-00003.safetensors | |
| - model-00002-of-00003.safetensors | |
| - model-00003-of-00003.safetensors | |
| output_dir: ${output_dir}/value | |
| model_type: REWARD | |
| seed: 53710 | |
| shuffle: true | |
| device: cuda | |
| batch_size: 64 | |
| num_steps: 10000 | |
| ppo_epochs: 2 | |
| ppo_batch_size: 32 | |
| gradient_accumulation_steps: 1 | |
| compile: false | |
| optimizer: | |
| _component_: bitsandbytes.optim.PagedAdamW | |
| lr: 0.0001 | |
| optimizer_in_bwd: true | |
| log_peak_memory_stats: true | |
| enable_activation_checkpointing: true | |
| dtype: bf16 | |
| forward_batch_size: 16 | |
| max_generated_tokens: 58 | |
| temperature: 0.7 | |
| top_k: null | |
| min_response_length: 18 | |
| penalise_no_eos: true | |
| reward_penalty: -3 | |
| stop_token_ids: | |
| - 2 | |
| - 28723 | |
| whiten_rewards: false | |
| gamma: 1 | |
| lmbda: 0.95 | |
| loss: | |
| _component_: torchtune.rlhf.loss.PPOLoss | |
| epsilon: 0.2 | |
| value_coeff: 0.1 | |
| value_clip_range: 0.2 | |
| kl_coeff: 0.01 | |
| wandb_logger: | |
| dir: ${prefix} | |
| entity: RADFAN | |
| project: FedPPO | |
| group: SelfReference | |
| name: Mistral-7B-LoRA-SelfRef-U13 | |
| log_every_n_steps: 1 | |
| update_ref_policy_every_n_steps: 13 | |