From 9fdb21e9de0bc073d7a1d9b1ae8c1c0864fa682c Mon Sep 17 00:00:00 2001 From: Matthew Douglas <38992547+matthewdouglas@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:49:05 -0400 Subject: [PATCH] Update Docker image builds for torch 2.7+cu126 (#2514) * Update Docker image builds for torch 2.7+cu126 * Remove bnb multi-source dockerfile --- .github/workflows/build_docker_images.yml | 34 ----------- .github/workflows/nightly-bnb.yml | 4 +- docker/README.md | 3 - docker/peft-gpu-bnb-latest/Dockerfile | 4 +- docker/peft-gpu-bnb-multi-source/Dockerfile | 68 --------------------- docker/peft-gpu-bnb-source/Dockerfile | 4 +- 6 files changed, 6 insertions(+), 111 deletions(-) delete mode 100644 docker/peft-gpu-bnb-multi-source/Dockerfile diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml index ce557730..f9e854c5 100644 --- a/.github/workflows/build_docker_images.yml +++ b/.github/workflows/build_docker_images.yml @@ -148,37 +148,3 @@ jobs: status: ${{ job.status }} slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} - latest-cuda-bnb-source-multi: - name: "Latest Peft GPU + bnb (multi-backend) source [accelerate / peft / transformers source]" - runs-on: - group: aws-general-8-plus - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Check out code - uses: actions/checkout@v3 - with: - persist-credentials: false - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and Push GPU - uses: docker/build-push-action@v4 - with: - context: ./docker/peft-gpu-bnb-multi-source - push: true - tags: huggingface/peft-gpu-bnb-multi-source - - - name: Post to Slack - if: always() - uses: huggingface/hf-workflows/.github/actions/post-slack@main - with: - slack_channel: ${{ env.CI_SLACK_CHANNEL }} - title: 🤗 Results of the PEFT-GPU (bnb source multi-backend / HF latest) docker build - status: ${{ job.status }} - slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} - - diff --git a/.github/workflows/nightly-bnb.yml b/.github/workflows/nightly-bnb.yml index 3c93d182..d75df16f 100644 --- a/.github/workflows/nightly-bnb.yml +++ b/.github/workflows/nightly-bnb.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"] + docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest"] runs-on: group: aws-g6-4xlarge-plus env: @@ -145,7 +145,7 @@ jobs: strategy: fail-fast: false matrix: - docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"] + docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest"] runs-on: group: aws-g6-12xlarge-plus env: diff --git a/docker/README.md b/docker/README.md index 07f60957..193f11b7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,6 +6,3 @@ Here we store all PEFT Docker images used in our testing infrastructure. We use - `peft-gpu`: PEFT complied for NVIDIA GPUs with all other HF libraries installed on main branch - `peft-gpu-bnb-source`: PEFT complied for NVIDIA GPUs with `bitsandbytes` and all other HF libraries installed from main branch - `peft-gpu-bnb-latest`: PEFT complied for NVIDIA GPUs with `bitsandbytes` complied from main and all other HF libraries installed from latest PyPi -- `peft-gpu-bnb-multi-source`: PEFT complied for NVIDIA GPUs with `bitsandbytes` complied from `multi-backend` branch and all other HF libraries installed from main branch - -`peft-gpu-bnb-source` and `peft-gpu-bnb-multi-source` are essentially the same, with the only difference being `bitsandbytes` compiled on another branch. Make sure to propagate the changes you applied on one file to the other! diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index 0406c509..c8265ac5 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -31,7 +31,7 @@ RUN chsh -s /bin/bash SHELL ["/bin/bash", "-c"] # Stage 2 -FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS build-image +FROM nvidia/cuda:12.6.3-devel-ubuntu22.04 AS build-image COPY --from=compile-image /opt/conda /opt/conda ENV PATH /opt/conda/bin:$PATH @@ -56,7 +56,7 @@ RUN source activate peft && \ peft \ optimum \ auto-gptq && \ - git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ + git clone https://github.com/bitsandbytes-foundation/bitsandbytes && cd bitsandbytes && \ cmake -B . -DCOMPUTE_BACKEND=cuda -S . && \ cmake --build . && \ pip install -e . && \ diff --git a/docker/peft-gpu-bnb-multi-source/Dockerfile b/docker/peft-gpu-bnb-multi-source/Dockerfile deleted file mode 100644 index 5ca63869..00000000 --- a/docker/peft-gpu-bnb-multi-source/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# Builds GPU docker image of PyTorch -# Uses multi-staged approach to reduce size -# Stage 1 -# Use base conda image to reduce time -FROM continuumio/miniconda3:latest AS compile-image -# Specify py version -ENV PYTHON_VERSION=3.11 -# Install apt libs - copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile -RUN apt-get update && \ - apt-get install -y curl git wget software-properties-common git-lfs && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Install audio-related libraries -RUN apt-get update && \ - apt install -y ffmpeg - -RUN apt install -y libsndfile1-dev -RUN git lfs install - -# Create our conda env - copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile -RUN conda create --name peft python=${PYTHON_VERSION} ipython jupyter pip -RUN python3 -m pip install --no-cache-dir --upgrade pip - -# Below is copied from https://github.com/huggingface/accelerate/blob/main/docker/accelerate-gpu/Dockerfile -# We don't install pytorch here yet since CUDA isn't available -# instead we use the direct torch wheel -ENV PATH /opt/conda/envs/peft/bin:$PATH -# Activate our bash shell -RUN chsh -s /bin/bash -SHELL ["/bin/bash", "-c"] - -# Stage 2 -FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS build-image -COPY --from=compile-image /opt/conda /opt/conda -ENV PATH /opt/conda/bin:$PATH - -RUN chsh -s /bin/bash -SHELL ["/bin/bash", "-c"] - -# Install apt libs -RUN apt-get update && \ - apt-get install -y curl git wget cmake && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Activate the conda env and install transformers + accelerate from source -# Also clone BNB and build it from source. -RUN source activate peft && \ - python3 -m pip install -U --no-cache-dir \ - librosa \ - "soundfile>=0.12.1" \ - scipy \ - git+https://github.com/huggingface/transformers \ - git+https://github.com/huggingface/accelerate \ - peft[test]@git+https://github.com/huggingface/peft \ - optimum \ - auto-gptq && \ - git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && git checkout multi-backend-refactor && \ - cmake -B . -DCOMPUTE_BACKEND=cuda -S . && \ - cmake --build . && \ - pip install -e . && \ - pip freeze | grep bitsandbytes - -RUN echo "source activate peft" >> ~/.profile - -# Activate the virtualenv -CMD ["/bin/bash"] diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index d4956bb3..afaa6193 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -31,7 +31,7 @@ RUN chsh -s /bin/bash SHELL ["/bin/bash", "-c"] # Stage 2 -FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS build-image +FROM nvidia/cuda:12.6.3-devel-ubuntu22.04 AS build-image COPY --from=compile-image /opt/conda /opt/conda ENV PATH /opt/conda/bin:$PATH @@ -56,7 +56,7 @@ RUN source activate peft && \ peft[test]@git+https://github.com/huggingface/peft \ optimum \ auto-gptq && \ - git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ + git clone https://github.com/bitsandbytes-foundation/bitsandbytes && cd bitsandbytes && \ cmake -B . -DCOMPUTE_BACKEND=cuda -S . && \ cmake --build . && \ pip install -e . && \