mirror of
https://github.com/deepspeedai/DeepSpeed.git
synced 2025-10-20 15:33:51 +08:00
a12de38db6
This is an initial effort to migrate CI unto Modal infra. This PR creates two new workflows that run on Modal 1. modal-torch-latest: a subset of nv-torch-latest-v100 that includes `tests/unit/runtime/zero/test_zero.py`. 2. modal-accelerate: a full copy of nv-accelerate-v100. Follow up PRs will selectively migrate relevant workflows onto Modal. --------- Signed-off-by: Olatunji Ruwase <tunji.ruwase@snowflake.com> Signed-off-by: Olatunji Ruwase <tjruwase@gmail.com> Signed-off-by: Tunji Ruwase <tunji.ruwase@snowflake.com> Co-authored-by: Stas Bekman <stas00@users.noreply.github.com> Co-authored-by: Logan Adams <loadams@microsoft.com> Co-authored-by: Logan Adams <114770087+loadams@users.noreply.github.com> Co-authored-by: Olatunji Ruwase <tjruwase@gmail.com> Co-authored-by: Stas Bekman <stas.bekman@snowflake.com>
44 lines
1.5 KiB
Python
44 lines
1.5 KiB
Python
# Copyright (c) Snowflake.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# DeepSpeed Team
|
|
|
|
from pathlib import Path
|
|
|
|
import modal
|
|
|
|
ROOT_PATH = Path(__file__).parents[1]
|
|
|
|
# yapf: disable
|
|
image = (modal.Image
|
|
.from_registry("pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel", add_python="3.10")
|
|
.run_commands("apt update && apt install -y libaio-dev")
|
|
.apt_install("git")
|
|
.run_commands("uv pip install --system --compile-bytecode datasets==3.6.0")
|
|
.run_commands(
|
|
"git clone https://github.com/huggingface/accelerate && \
|
|
uv pip install --system --compile-bytecode ./accelerate[testing]"
|
|
)
|
|
.pip_install_from_requirements(ROOT_PATH / "requirements/requirements.txt", gpu="any")
|
|
.pip_install_from_requirements(ROOT_PATH / "requirements/requirements-dev.txt", gpu="any")
|
|
.add_local_dir(ROOT_PATH , remote_path="/root/", copy=True)
|
|
.run_commands("pip install /root")
|
|
.add_local_dir(ROOT_PATH / "accelerator", remote_path="/root/deepspeed/accelerator")
|
|
.add_local_dir(ROOT_PATH / "csrc", remote_path="/root/deepspeed/ops/csrc")
|
|
.add_local_dir(ROOT_PATH / "op_builder", remote_path="/root/deepspeed/ops/op_builder")
|
|
)
|
|
|
|
app = modal.App("deepspeedai-accelerate-ci", image=image)
|
|
|
|
@app.function(
|
|
gpu="l40s:1",
|
|
timeout=1800,
|
|
)
|
|
def pytest():
|
|
import subprocess
|
|
subprocess.run(
|
|
"pytest /accelerate/tests/deepspeed".split(),
|
|
check=True,
|
|
cwd=ROOT_PATH / ".",
|
|
)
|