mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
9fedf41b60
Fixes #125526 [#1811](https://github.com/pytorch/builder/issues/1811) Adopt syntax=docker/dockerfile:1 whcih has been stable since 2018, while still best practice to declare in 2024. - Syntax features dependent upon the [syntax directive version are documented here](https://hub.docker.com/r/docker/dockerfile). - While you can set a fixed minor version, [Docker officially advises to only pin the major version] ``` (https://docs.docker.com/build/dockerfile/frontend/#stable-channel): We recommend using docker/dockerfile:1, which always points to the latest stable release of the version 1 syntax, and receives both "minor" and "patch" updates for the version 1 release cycle. BuildKit automatically checks for updates of the syntax when performing a build, making sure you are using the most current version. ``` **Support for building with Docker prior to v23 (released on Feb 2023)** NOTE: 18.06 may not be the accurate minimum version for using docker/dockerfile:1, according to the [DockerHub tag history](https://hub.docker.com/layers/docker/dockerfile/1.0/images/sha256-92f5351b2fca8f7e2f452aa9aec1c34213cdd2702ca92414eee6466fab21814a?context=explore) 1.0 of the syntax seems to be from Dec 2018, which is probably why docker/dockerfile:experimental was paired with it in this file. Personally, I'd favor only supporting builds with Docker v23. This is only relevant for someone building this Dockerfile locally, the user could still extend the already built and published image from a registry on older versions of Docker without any concern for this directive which only applies to building this Dockerfile, not images that extend it. However if you're reluctant, you may want to refer others to [this Docker docs page](https://docs.docker.com/build/buildkit/#getting-started) where they should only need the ENV DOCKER_BUILDKIT=1, presumably the requirement for experimental was dropped with syntax=docker/dockerfile:1 with releases of Docker since Dec 2018. Affected users can often quite easily install a newer version of Docker on their OS, as per Dockers official guidance (usually via including an additional repo to the package manager). **Reference links** Since one of these was already included in the inline note (now a broken link), I've included relevant links mentioned above. You could alternatively rely on git blame with a commit message referencing the links or this PR for more information. Feel free to remove any of the reference links, they're mostly only relevant to maintainers to be aware of (which this PR itself has detailed adequately above). Pull Request resolved: https://github.com/pytorch/pytorch/pull/125632 Approved by: https://github.com/malfet
107 lines
4.0 KiB
Docker
107 lines
4.0 KiB
Docker
# syntax=docker/dockerfile:1
|
|
|
|
# NOTE: Building this image require's docker version >= 23.0.
|
|
#
|
|
# For reference:
|
|
# - https://docs.docker.com/build/dockerfile/frontend/#stable-channel
|
|
|
|
ARG BASE_IMAGE=ubuntu:22.04
|
|
ARG PYTHON_VERSION=3.11
|
|
|
|
FROM ${BASE_IMAGE} as dev-base
|
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
ca-certificates \
|
|
ccache \
|
|
cmake \
|
|
curl \
|
|
git \
|
|
libjpeg-dev \
|
|
libpng-dev && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
RUN /usr/sbin/update-ccache-symlinks
|
|
RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache
|
|
ENV PATH /opt/conda/bin:$PATH
|
|
|
|
FROM dev-base as conda
|
|
ARG PYTHON_VERSION=3.11
|
|
# Automatically set by buildx
|
|
ARG TARGETPLATFORM
|
|
# translating Docker's TARGETPLATFORM into miniconda arches
|
|
RUN case ${TARGETPLATFORM} in \
|
|
"linux/arm64") MINICONDA_ARCH=aarch64 ;; \
|
|
*) MINICONDA_ARCH=x86_64 ;; \
|
|
esac && \
|
|
curl -fsSL -v -o ~/miniconda.sh -O "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${MINICONDA_ARCH}.sh"
|
|
COPY requirements.txt .
|
|
# Manually invoke bash on miniconda script per https://github.com/conda/conda/issues/10431
|
|
RUN chmod +x ~/miniconda.sh && \
|
|
bash ~/miniconda.sh -b -p /opt/conda && \
|
|
rm ~/miniconda.sh && \
|
|
/opt/conda/bin/conda install -y python=${PYTHON_VERSION} cmake conda-build pyyaml numpy ipython && \
|
|
/opt/conda/bin/python -mpip install -r requirements.txt && \
|
|
/opt/conda/bin/conda clean -ya
|
|
|
|
FROM dev-base as submodule-update
|
|
WORKDIR /opt/pytorch
|
|
COPY . .
|
|
RUN git submodule update --init --recursive
|
|
|
|
FROM conda as build
|
|
ARG CMAKE_VARS
|
|
WORKDIR /opt/pytorch
|
|
COPY --from=conda /opt/conda /opt/conda
|
|
COPY --from=submodule-update /opt/pytorch /opt/pytorch
|
|
RUN make triton
|
|
RUN --mount=type=cache,target=/opt/ccache \
|
|
export eval ${CMAKE_VARS} && \
|
|
TORCH_CUDA_ARCH_LIST="7.0 7.2 7.5 8.0 8.6 8.7 8.9 9.0 9.0a" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
|
|
CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
|
|
python setup.py install
|
|
|
|
FROM conda as conda-installs
|
|
ARG PYTHON_VERSION=3.11
|
|
ARG CUDA_VERSION=12.1
|
|
ARG CUDA_CHANNEL=nvidia
|
|
ARG INSTALL_CHANNEL=pytorch-nightly
|
|
# Automatically set by buildx
|
|
# Note conda needs to be pinned to 23.5.2 see: https://github.com/pytorch/pytorch/issues/106470
|
|
RUN /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -y python=${PYTHON_VERSION} conda=23.5.2
|
|
ARG TARGETPLATFORM
|
|
|
|
# On arm64 we can only install wheel packages.
|
|
RUN case ${TARGETPLATFORM} in \
|
|
"linux/arm64") pip install --extra-index-url https://download.pytorch.org/whl/cpu/ torch torchvision torchaudio ;; \
|
|
*) /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch torchvision torchaudio "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" ;; \
|
|
esac && \
|
|
/opt/conda/bin/conda clean -ya
|
|
RUN /opt/conda/bin/pip install torchelastic
|
|
|
|
FROM ${BASE_IMAGE} as official
|
|
ARG PYTORCH_VERSION
|
|
ARG TRITON_VERSION
|
|
ARG TARGETPLATFORM
|
|
ARG CUDA_VERSION
|
|
LABEL com.nvidia.volumes.needed="nvidia_driver"
|
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
libjpeg-dev \
|
|
libpng-dev \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
COPY --from=conda-installs /opt/conda /opt/conda
|
|
RUN if test -n "${TRITON_VERSION}" -a "${TARGETPLATFORM}" != "linux/arm64"; then \
|
|
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends gcc; \
|
|
rm -rf /var/lib/apt/lists/*; \
|
|
fi
|
|
ENV PATH /opt/conda/bin:$PATH
|
|
ENV NVIDIA_VISIBLE_DEVICES all
|
|
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
|
|
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
|
|
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH
|
|
ENV PYTORCH_VERSION ${PYTORCH_VERSION}
|
|
WORKDIR /workspace
|
|
|
|
FROM official as dev
|
|
# Should override the already installed version from the official-image stage
|
|
COPY --from=build /opt/conda /opt/conda
|