mirror of
https://github.com/volcengine/verl.git
synced 2025-10-20 13:43:50 +08:00
bd756c15c8
This PR allows users to pass all vllm/sglang engine args and optimizes qwen3 rollout speed through vllm Engine argument. 1. deprecate the default value of previous engine_kwargs 2. pass all the engine_kwargs to vllm/sglang engine 3. optimize Qwen3-235B rollout speed by setting TP=8 and enabling expert parallel. From top to bottom: tp=16 without EP, tp=8 without EP and tp=8 with EP. <img width="1000" height="808" alt="image" src="https://github.com/user-attachments/assets/6b096be4-3896-4e96-8916-d8d6e13a58cc" /> PS: The DeepSeek-V3's rollout slows down after enabling expert parallelism.
52 lines
2.3 KiB
Bash
52 lines
2.3 KiB
Bash
set -x
|
|
ENGINE=${1:-vllm}
|
|
|
|
# Some models are optimized by vllm ascend. While in some case, e.g. rlhf training,
|
|
# the optimized model may not be suitable. In this case, set this value to 0 to disable the optimized model.
|
|
export USE_OPTIMIZED_MODEL=0
|
|
|
|
python3 -m verl.trainer.main_ppo \
|
|
algorithm.adv_estimator=grpo \
|
|
data.train_files=$HOME/data/geo3k/train.parquet \
|
|
data.val_files=$HOME/data/geo3k/test.parquet \
|
|
data.train_batch_size=512 \
|
|
data.max_prompt_length=1024 \
|
|
data.max_response_length=2048 \
|
|
data.filter_overlong_prompts=True \
|
|
data.truncation='error' \
|
|
data.image_key=images \
|
|
actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-7B-Instruct \
|
|
actor_rollout_ref.actor.optim.lr=1e-6 \
|
|
actor_rollout_ref.model.use_remove_padding=True \
|
|
actor_rollout_ref.actor.ppo_mini_batch_size=32 \
|
|
actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \
|
|
actor_rollout_ref.actor.use_kl_loss=True \
|
|
actor_rollout_ref.actor.kl_loss_coef=0.01 \
|
|
actor_rollout_ref.actor.kl_loss_type=low_var_kl \
|
|
actor_rollout_ref.actor.entropy_coeff=0 \
|
|
actor_rollout_ref.actor.use_torch_compile=False \
|
|
actor_rollout_ref.model.enable_gradient_checkpointing=True \
|
|
actor_rollout_ref.actor.fsdp_config.param_offload=False \
|
|
actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
|
|
actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \
|
|
actor_rollout_ref.rollout.tensor_model_parallel_size=4 \
|
|
actor_rollout_ref.rollout.name=$ENGINE \
|
|
+actor_rollout_ref.rollout.engine_kwargs.vllm.disable_mm_preprocessor_cache=True \
|
|
actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \
|
|
actor_rollout_ref.rollout.enable_chunked_prefill=False \
|
|
actor_rollout_ref.rollout.enforce_eager=True \
|
|
actor_rollout_ref.rollout.free_cache_engine=True \
|
|
actor_rollout_ref.rollout.n=5 \
|
|
actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \
|
|
actor_rollout_ref.ref.fsdp_config.param_offload=True \
|
|
algorithm.use_kl_in_reward=False \
|
|
trainer.critic_warmup=0 \
|
|
trainer.logger=console \
|
|
trainer.project_name='verl_grpo_example_geo3k' \
|
|
trainer.experiment_name='qwen2_5_vl_7b_function_rm' \
|
|
trainer.n_gpus_per_node=16 \
|
|
trainer.nnodes=1 \
|
|
trainer.save_freq=-1 \
|
|
trainer.test_freq=-1 \
|
|
trainer.total_epochs=15 \
|
|
trainer.device=npu $@ |