data:
  tokenizer: null
  train_files: ~/data/rlhf/gsm8k/train.parquet
  val_files: ~/data/rlhf/gsm8k/test.parquet
  prompt_key: prompt
  reward_fn_key: data_source
  max_prompt_length: 512
  max_response_length: 512
  train_batch_size: 1024
  val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves
  return_raw_input_ids: False  # This should be set to true when the tokenizer between policy and rm differs
  return_raw_chat: False
  return_full_prompt: False
  shuffle: True
  filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You cat set the filter_overlong_prompts_workers to use multiprocessing to speed up.
  filter_overlong_prompts_workers: 1
  truncation: error
  trust_remote_code: False  # main_ppo will check this config to determine whether to use remote code for tokenizer
  custom_cls:
      path: null
      name: null

actor_rollout_ref:
  hybrid_engine: True
  model:
    path: ~/models/deepseek-llm-7b-chat
    external_lib: null
    override_config:
      model_config: {}
      moe_config:
        freeze_moe_router: False
    enable_gradient_checkpointing: False
    gradient_checkpointing_kwargs:
      ## Activation Checkpointing
      activations_checkpoint_method: null # 'uniform', 'block'; not used with 'selective'
      # 'uniform' divides the total number of transformer layers and checkpoints the input activation of each chunk
      # 'block' checkpoints the specified number of layers per pipeline stage at the specified granularity
      activations_checkpoint_granularity: null # 'selective' or 'full'
      # 'full' will checkpoint the entire transformer layer and 'selective' only checkpoints memory intensive part of attention
      activations_checkpoint_num_layers: null # not used with 'selective'
    trust_remote_code: False
  actor:
    strategy: megatron  # This is for backward-compatibility
    ppo_mini_batch_size: 256
    ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu
    ppo_micro_batch_size_per_gpu: null
    use_dynamic_bsz: False
    ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length}
    use_torch_compile: True # False to disable torch compile
    # pg_losses2 = -advantages * torch.clamp(ratio, 1 - cliprange_low, 1 + cliprange_high)
    clip_ratio: 0.2 # default value if clip_ratio_low and clip_ratio_high are not specified
    clip_ratio_low: 0.2
    clip_ratio_high: 0.2
    clip_ratio_c: 3.0 # lower bound of the value for Dual-clip PPO from https://arxiv.org/pdf/1912.09729
    loss_agg_mode: "token-mean" # / "seq-mean-token-sum" / "seq-mean-token-mean"
    # NOTE: "token-mean" is the default behavior
    entropy_coeff: 0
    use_kl_loss: False # True for GRPO
    kl_loss_coef: 0.001 # for grpo
    kl_loss_type: low_var_kl # for grpo
    ppo_epochs: 1
    data_loader_seed: null
    shuffle: False
    optim:
      lr: 1e-6
      clip_grad: 1.0
      lr_warmup_steps: -1 # Prioritized. Negative values mean delegating to lr_warmup_steps_ratio.
      lr_warmup_steps_ratio: 0.  # the total steps will be injected during runtime
      min_lr_ratio: null   # only useful for warmup with cosine
      warmup_style: constant  # select from constant/cosine
      total_training_steps: -1  # must be override by program
      weight_decay: 0.01
    megatron:
      param_offload: False
      grad_offload: False
      optimizer_offload: False
      tensor_model_parallel_size: 1
      expert_model_parallel_size: 1
      expert_tensor_parallel_size: null
      pipeline_model_parallel_size: 1
      virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests
      context_parallel_size: 1
      sequence_parallel: True
      use_distributed_optimizer: True
      use_dist_checkpointing: False
      dist_checkpointing_path: null
      seed: 42
      override_transformer_config: {} # additional transformer config like: num_layers_in_first(/last)_pipeline_stage
    profile: # profile the actor model in `update_policy` 
      use_profile: False # open it when you want to profile the actor model
      profile_ranks: null # list, you can specify the ranks to profile
      step_start: -1 # start step in update_policy 
      step_end: -1 # end step 
      save_path: null # the path to save the profile result
    load_weight: True
    checkpoint:
      contents: ['model', 'optimizer', 'extra']  # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space
  ref:
    strategy: megatron
    use_torch_compile: ${actor_rollout_ref.actor.use_torch_compile}
    megatron:
      param_offload: False
      tensor_model_parallel_size: 1
      expert_model_parallel_size: 1
      expert_tensor_parallel_size: None
      pipeline_model_parallel_size: 1
      virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests
      context_parallel_size: 1
      sequence_parallel: True
      use_distributed_optimizer: False
      use_dist_checkpointing: False
      dist_checkpointing_path: null
      seed: ${actor_rollout_ref.actor.megatron.seed}
      override_transformer_config: ${actor_rollout_ref.actor.megatron.override_transformer_config}
    profile:
      use_profile: False
      profile_ranks: null
      step_start: -1
      step_end: -1
      save_path: null
    load_weight: True
    log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu
    log_prob_micro_batch_size_per_gpu: null
    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
  rollout:
    name: vllm
    mode: sync # sync: LLM, async: AsyncLLM
    temperature: 1.0
    top_k: -1 # 0 for hf rollout, -1 for vllm rollout
    top_p: 1
    prompt_length: ${data.max_prompt_length}  # for xperf_gpt
    response_length: ${data.max_response_length}
    # for vllm rollout
    dtype: bfloat16 # should align with FSDP
    gpu_memory_utilization: 0.5
    ignore_eos: False
    enforce_eager: True
    free_cache_engine: True
    load_format: dummy_megatron
    tensor_model_parallel_size: 1
    max_num_batched_tokens: 8192
    max_model_len: null
    max_num_seqs: 1024
    log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu
    log_prob_micro_batch_size_per_gpu: null
    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}
    disable_log_stats: True
    enable_chunked_prefill: False # could get higher throughput
    # for hf rollout
    do_sample: True
    layer_name_map:
      qkv_layer_name: qkv
      gate_proj_layer_name: gate_up
    # number of responses (i.e. num sample times)
    n: 1
    engine_kwargs: # inference engine parameters
      vllm:
        swap_space: null # null means "use the engine default value" (usually 4 GB), setting it to, e.g., 32 means 32 GB
      sglang:
        attention_backend: null # null means use the engine default value, available options: flashinfer, triton, flashmla
    val_kwargs:
      # sampling parameters for validation
      top_k: -1 # 0 for hf rollout, -1 for vllm rollout
      top_p: 1.0
      temperature: 0
      n: 1
      do_sample: False # default eager for validation
    multi_turn: 
      enable: False  # set to True for multi-turn tool interaction tasks; should set rollout.name to sglang as well
      max_turns: null  # null for no limit (default max_length // 3)
      tool_config_path: null  # null for no tool
      format: chatml  # chatml, more formats will be supported in the future

critic:
  rollout_n: ${actor_rollout_ref.rollout.n}
  strategy: megatron
  optim:
    lr: 1e-5
    clip_grad: 1.0
    lr_warmup_steps_ratio: 0.  # the total steps will be injected during runtime
    min_lr_ratio: null   # only useful for warmup with cosine
    warmup_style: constant  # select from constant/cosine
    total_training_steps: -1  # must be override by program
    weight_decay: 0.01
  model:
    path: ~/models/deepseek-llm-7b-chat
    tokenizer_path: ${actor_rollout_ref.model.path}
    override_config:
      model_config: {}
      moe_config:
        freeze_moe_router: False
    external_lib: ${actor_rollout_ref.model.external_lib}
    trust_remote_code: False
    enable_gradient_checkpointing: False
    gradient_checkpointing_kwargs:
      ## Activation Checkpointing
      activations_checkpoint_method: null
      activations_checkpoint_granularity: null
      activations_checkpoint_num_layers: null
  megatron:
    param_offload: False
    grad_offload: False
    optimizer_offload: False
    tensor_model_parallel_size: 1
    expert_model_parallel_size: 1
    expert_tensor_parallel_size: null
    pipeline_model_parallel_size: 1
    virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests
    context_parallel_size: 1
    sequence_parallel: True
    use_distributed_optimizer: True
    use_dist_checkpointing: False
    dist_checkpointing_path: null
    seed: ${actor_rollout_ref.actor.megatron.seed}
    override_transformer_config: ${actor_rollout_ref.actor.megatron.override_transformer_config}
  load_weight: True
  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
  ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu
  ppo_micro_batch_size_per_gpu: null
  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
  ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2
  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
  data_loader_seed: ${actor_rollout_ref.actor.data_loader_seed}
  shuffle: ${actor_rollout_ref.actor.shuffle}
  cliprange_value: 0.5
  kl_ctrl:
    type: fixed
    kl_coef: 0.001
  loss_agg_mode: ${actor_rollout_ref.actor.loss_agg_mode}
  checkpoint:
    contents: ['model', 'optimizer', 'extra']  # with 'hf_model' you can save whole model as hf format, now only use sharded model checkpoint to save space

reward_model:
  enable: False
  strategy: megatron
  megatron:
    param_offload: False
    tensor_model_parallel_size: 1
    expert_model_parallel_size: 1
    expert_tensor_parallel_size: null
    pipeline_model_parallel_size: 1
    virtual_pipeline_model_parallel_size: null # change VPP interface for parallelism tests
    context_parallel_size: 1
    sequence_parallel: True
    use_distributed_optimizer: False
    use_dist_checkpointing: False
    dist_checkpointing_path: null
    seed: ${actor_rollout_ref.actor.megatron.seed}
    override_transformer_config: {}
  model:
    input_tokenizer: ${actor_rollout_ref.model.path}  # set this to null if the chat template is identical
    path: ~/models/FsfairX-LLaMA3-RM-v0.1
    trust_remote_code: False
    external_lib: ${actor_rollout_ref.model.external_lib}
  load_weight: True
  micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu
  micro_batch_size_per_gpu: null
  use_dynamic_bsz: ${critic.use_dynamic_bsz}
  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
  max_length: null
  reward_manager: naive
  launch_reward_fn_async: False # custom reward function executed async on CPU, during log_prob
  sandbox_fusion:
    url: null # faas url to run code in cloud sandbox
    max_concurrent: 64 # max concurrent requests to sandbox
custom_reward_function:
  path: null
  name: compute_score

algorithm:
  gamma: 1.0
  lam: 1.0
  adv_estimator: gae
  norm_adv_by_std_in_grpo: True
  use_kl_in_reward: False
  kl_penalty: kl  # how to estimate kl divergence
  kl_ctrl:
    type: fixed
    kl_coef: 0.001
    horizon: 10000
    target_kl: 0.1
  use_pf_ppo: False
  pf_ppo:
    reweight_method: pow  # ["pow", "max_min", "max_random"]
    weight_pow: 2.0

trainer:
  balance_batch: True
  total_epochs: 30
  total_training_steps: null
  project_name: verl_examples
  experiment_name: gsm8k
  logger: ['console', 'wandb']
  log_val_generations: 0
  nnodes: 1
  n_gpus_per_node: 8
  save_freq: -1
  # auto: find the last ckpt to resume. If can't find, start from scratch
  resume_mode: auto # or disable or resume_path if resume_from_path is set
  resume_from_path: null
  del_local_ckpt_after_load: False
  val_before_train: True
  test_freq: -1
  critic_warmup: 0
  default_hdfs_dir: null
  default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name}
  max_actor_ckpt_to_keep: null
  max_critic_ckpt_to_keep: null
  # The timeout for ray worker group to wait for the register center to be ready
  ray_wait_register_center_timeout: 300
  device: cuda

ray_init:
  num_cpus: null # `None` means using all CPUs, which might cause hang if limited in systems like SLURM. Please set to a number allowed then.
  timeline_json_file: null
