mirror of
https://github.com/volcengine/verl.git
synced 2025-10-20 13:43:50 +08:00
6ff2b43d13
### What does this PR do? Solve https://github.com/volcengine/verl/pull/3530#issuecomment-3332840437
119 lines
3.4 KiB
YAML
119 lines
3.4 KiB
YAML
name: e2e_spin
|
|
|
|
on:
|
|
# Trigger the workflow on push or pull request,
|
|
# but only for the main branch
|
|
push:
|
|
branches:
|
|
- main
|
|
- v0.*
|
|
paths:
|
|
- "**/*.py"
|
|
# Other entrypoints
|
|
- "!examples/**"
|
|
- "!tests/**"
|
|
- "!verl/trainer/main_*.py"
|
|
- "!verl/trainer/fsdp_sft_trainer.py"
|
|
# Other recipes
|
|
- "!recipe/**"
|
|
# Megatron
|
|
- "!verl/workers/**/megatron_*.py"
|
|
# Home
|
|
- "recipe/spin"
|
|
# Entrypoints
|
|
- ".github/workflows/e2e_spin.yml"
|
|
- "examples/data_preprocess/gsm8k.py"
|
|
- "tests/special_e2e/run_spin.sh"
|
|
- "!examples"
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- v0.*
|
|
paths:
|
|
- "**/*.py"
|
|
# Other entrypoints
|
|
- "!examples/**"
|
|
- "!tests/**"
|
|
- "!verl/trainer/main_*.py"
|
|
- "!verl/trainer/fsdp_sft_trainer.py"
|
|
# Other recipes
|
|
- "!recipe/**"
|
|
# Megatron
|
|
- "!verl/workers/**/megatron_*.py"
|
|
# Home
|
|
- "recipe/spin"
|
|
# Entrypoints
|
|
- ".github/workflows/e2e_spin.yml"
|
|
- "examples/data_preprocess/gsm8k.py"
|
|
- "tests/special_e2e/run_spin.sh"
|
|
- "!examples"
|
|
|
|
# Declare permissions just read content.
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
IMAGE: "verl-ci-cn-beijing.cr.volces.com/verlai/verl:app-verl0.6-transformers4.56.1-sglang0.5.2-mcore0.13.0-te2.2"
|
|
DYNAMIC_RUNNER_ENDPOINT: "https://sd10g3clalm04ug7alq90.apigateway-cn-beijing.volceapi.com/runner"
|
|
|
|
# Cancel jobs on the same ref if a new one is triggered
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
|
|
|
jobs:
|
|
setup:
|
|
if: github.repository_owner == 'volcengine'
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
runner-label: ${{ steps.create-runner.outputs.runner-label }}
|
|
mlp-task-id: ${{ steps.create-runner.outputs.mlp-task-id }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- id: create-runner
|
|
uses: volcengine/vemlp-github-runner@v1
|
|
with:
|
|
mode: "create"
|
|
faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
|
|
mlp-image: "${{ env.IMAGE }}"
|
|
|
|
e2e_spin:
|
|
needs: setup
|
|
runs-on: [ "${{ needs.setup.outputs.runner-label || 'L20x8' }}" ]
|
|
timeout-minutes: 40 # Increase this timeout value as needed
|
|
env:
|
|
HTTP_PROXY: ${{ secrets.PROXY_HTTP }}
|
|
HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
|
|
NO_PROXY: "localhost,127.0.0.1,hf-mirror.com"
|
|
HF_ENDPOINT: "https://hf-mirror.com"
|
|
HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
|
|
steps:
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Install the current repository
|
|
run: |
|
|
pip3 install -e .[test,gpu,sglang]
|
|
- name: Prepare GSM8K dataset
|
|
run: |
|
|
python3 examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/models/hf_data/gsm8k
|
|
- name: Running the E2E test with the spin algorithm
|
|
run: |
|
|
ray stop --force
|
|
bash tests/special_e2e/run_spin.sh
|
|
|
|
cleanup:
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
[
|
|
setup,
|
|
e2e_spin
|
|
]
|
|
if: always()
|
|
steps:
|
|
- id: destroy-runner
|
|
uses: volcengine/vemlp-github-runner@v1
|
|
with:
|
|
mode: "destroy"
|
|
faas-url: "${{ env.DYNAMIC_RUNNER_ENDPOINT }}"
|
|
mlp-task-id: "${{ needs.setup.outputs.mlp-task-id }}" |