From f4e20470741a523da00fee7bea33bb986c8ef85b Mon Sep 17 00:00:00 2001 From: xvxuopop <127376094+xvxuopop@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:48:24 +0800 Subject: [PATCH] [model, ci] feat: add qwen3-8b ppo script on ASCEND NPU (#3502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What does this PR do? add examples/ppo_trainer/run_qwen3-8b_npu.sh > Add **concise** overview of what this PR aims to achieve or accomplish. Reference related GitHub issues and PRs that help with the review. ### Checklist Before Starting - [ ] Search for similar PRs. Paste at least one query link here: ... - [ ] Format the PR title as `[{modules}] {type}: {description}` (This will be checked by the CI) - `{modules}` include `fsdp`, `megatron`, `sglang`, `vllm`, `rollout`, `trainer`, `ci`, `training_utils`, `recipe`, `hardware`, `deployment`, `ray`, `worker`, `single_controller`, `misc`, `perf`, `model`, `algo`, `env`, `tool`, `ckpt`, `doc`, `data` - If this PR involves multiple modules, separate them with `,` like `[megatron, fsdp, doc]` - `{type}` is in `feat`, `fix`, `refactor`, `chore`, `test` - If this PR breaks any API (CLI arguments, config, function signature, etc.), add `[BREAKING]` to the beginning of the title. - Example: `[BREAKING][fsdp, megatron] feat: dynamic batching` ### Test > For changes that can not be tested by CI (e.g., algorithm implementation, new model support), validate by experiment(s) and show results like training curve plots, evaluation results, etc. ### API and Usage Example > Demonstrate how the API changes if any, and provide usage example(s) if possible. ```python # Add code snippet or script demonstrating how to use this ``` ### Design & Code Changes > Demonstrate the high-level design if this PR is complex, and list the specific changes. ### Checklist Before Submitting > [!IMPORTANT] > Please check all the following items before requesting a review, otherwise the reviewer might deprioritize this PR for review. - [ ] Read the [Contribute Guide](https://github.com/volcengine/verl/blob/main/CONTRIBUTING.md). - [ ] Apply [pre-commit checks](https://github.com/volcengine/verl/blob/main/CONTRIBUTING.md#code-linting-and-formatting): `pre-commit install && pre-commit run --all-files --show-diff-on-failure --color=always` - [ ] Add / Update [the documentation](https://github.com/volcengine/verl/tree/main/docs). - [ ] Add unit or end-to-end test(s) to [the CI workflow](https://github.com/volcengine/verl/tree/main/.github/workflows) to cover all the code. If not feasible, explain why: ... - [ ] Once your PR is ready for CI, send a message in [the `ci-request` channel](https://verl-project.slack.com/archives/C091TCESWB1) in [the `verl` Slack workspace](https://join.slack.com/t/verl-project/shared_invite/zt-3855yhg8g-CTkqXu~hKojPCmo7k_yXTQ). (If not accessible, please try [the Feishu group (飞书群)](https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=772jd4f1-cd91-441e-a820-498c6614126a).) --- .github/workflows/e2e_ascend.yml | 5 ++ docs/ascend_tutorial/ascend_quick_start.rst | 2 + examples/ppo_trainer/run_qwen3-8b_npu.sh | 55 +++++++++++++++++++++ tests/special_npu/run_qwen3_06b_ppo.sh | 51 +++++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 examples/ppo_trainer/run_qwen3-8b_npu.sh create mode 100644 tests/special_npu/run_qwen3_06b_ppo.sh diff --git a/.github/workflows/e2e_ascend.yml b/.github/workflows/e2e_ascend.yml index abf85ea16..3e760e21c 100644 --- a/.github/workflows/e2e_ascend.yml +++ b/.github/workflows/e2e_ascend.yml @@ -148,6 +148,11 @@ jobs: ray stop --force bash tests/special_npu/run_qwen3_06b_grpo.sh rm -rf $HOME/ckpts + - name: Running gsm8k e2e qwen3 training tests with PPO on ASCEND NPU + run: | + ray stop --force + bash tests/special_npu/run_qwen3_06b_ppo.sh + rm -rf $HOME/ckpts - name: Running gsm8k e2e training tests with GRPO MindSpeed on ASCEND NPU run: | ray stop --force diff --git a/docs/ascend_tutorial/ascend_quick_start.rst b/docs/ascend_tutorial/ascend_quick_start.rst index 2c2f91e50..4b311fcfe 100644 --- a/docs/ascend_tutorial/ascend_quick_start.rst +++ b/docs/ascend_tutorial/ascend_quick_start.rst @@ -193,6 +193,8 @@ vllm & vllm-ascend +-----------+-------------------------+-------------+-------------------+-------------------+-------------------+--------------------------+ | DAPO | Qwen3-30B-base | 1.08% | pending | FSDP | vllm-ascend | Atlas 200T A2 Box16 | +-----------+-------------------------+-------------+-------------------+-------------------+-------------------+--------------------------+ +| PPO | Qwen3-8B | 4.49% | 0.874 | FSDP | vllm-ascend | Atlas 900 A2 PODc | ++-----------+-------------------------+-------------+-------------------+-------------------+-------------------+--------------------------+ **表2** SFT类算法 diff --git a/examples/ppo_trainer/run_qwen3-8b_npu.sh b/examples/ppo_trainer/run_qwen3-8b_npu.sh new file mode 100644 index 000000000..40fb751e6 --- /dev/null +++ b/examples/ppo_trainer/run_qwen3-8b_npu.sh @@ -0,0 +1,55 @@ +set -x + +export VLLM_USE_V1=1 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/dapo-math-17k.parquet \ + data.val_files=$HOME/data/dapo-math-17k.parquet \ + data.train_batch_size=256 \ + data.max_prompt_length=2000 \ + data.max_response_length=12000 \ + data.shuffle=False \ + actor_rollout_ref.model.path=Qwen/Qwen3-8B \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \ + actor_rollout_ref.rollout.max_num_batched_tokens=14000 \ + actor_rollout_ref.rollout.max_num_seqs=64 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.enforce_eager=False \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen3-8B \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=1 \ + critic.ulysses_sequence_parallel_size=2 \ + critic.model.fsdp_config.param_offload=True \ + critic.model.fsdp_config.optimizer_offload=True \ + critic.use_dynamic_bsz=True \ + trainer.critic_warmup=0 \ + trainer.logger=console \ + trainer.project_name='verl_example_dapo_math_17k' \ + trainer.experiment_name='qwen3_8b_fsdp' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=20 \ + trainer.test_freq=-1 \ + trainer.val_before_train=False \ + trainer.device=npu \ + trainer.max_actor_ckpt_to_keep=1 \ + trainer.max_critic_ckpt_to_keep=1 \ + trainer.total_training_steps=100 $@ \ No newline at end of file diff --git a/tests/special_npu/run_qwen3_06b_ppo.sh b/tests/special_npu/run_qwen3_06b_ppo.sh new file mode 100644 index 000000000..de7b0e687 --- /dev/null +++ b/tests/special_npu/run_qwen3_06b_ppo.sh @@ -0,0 +1,51 @@ +set -x + +export VLLM_USE_V1=1 + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=gae \ + data.train_files=$HOME/data/gsm8k/train.parquet \ + data.val_files=$HOME/data/gsm8k/test.parquet \ + data.train_batch_size=128 \ + data.max_prompt_length=512 \ + data.max_response_length=128 \ + data.shuffle=False \ + actor_rollout_ref.model.path=Qwen/Qwen3-0.6B \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.ppo_mini_batch_size=64 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.actor.fsdp_config.param_offload=True \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=2 \ + actor_rollout_ref.actor.use_dynamic_bsz=True \ + actor_rollout_ref.actor.use_torch_compile=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.8 \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=True \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.enforce_eager=False \ + critic.optim.lr=1e-5 \ + critic.model.use_remove_padding=True \ + critic.model.path=Qwen/Qwen3-0.6B \ + critic.model.enable_gradient_checkpointing=True \ + critic.ppo_micro_batch_size_per_gpu=8 \ + critic.ulysses_sequence_parallel_size=2 \ + critic.model.fsdp_config.param_offload=True \ + critic.model.fsdp_config.optimizer_offload=True \ + critic.use_dynamic_bsz=True \ + trainer.critic_warmup=0 \ + trainer.logger='["console"]' \ + trainer.project_name='verl_ppo_example_gsm8k_qwen3' \ + trainer.experiment_name='qwen3_06b_fsdp' \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes=1 \ + trainer.save_freq=-1 \ + trainer.test_freq=5 \ + trainer.total_epochs=1 \ + trainer.total_training_steps=2 \ + trainer.device=npu $@