Build docker images for tokenizers main

.github/workflows/build-docker-images.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - docker-image*
+      - test-tokenizers-*
   repository_dispatch:
   workflow_call:
     inputs:
@@ -42,36 +43,7 @@ jobs:
           build-args: |
             REF=main
           push: true
-          tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }}
-
-  latest-with-torch-nightly-docker:
-    name: "Nightly PyTorch + Stable TensorFlow"
-    # Push CI doesn't need this image
-    if: inputs.image_postfix != '-push-ci'
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Check out code
-        uses: actions/checkout@v2
-      -
-        name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-      -
-        name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker/transformers-all-latest-gpu
-          build-args: |
-            REF=main
-            PYTORCH=pre
-          push: true
-          tags: huggingface/transformers-all-latest-torch-nightly-gpu
+          tags: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
 
   latest-torch-deepspeed-docker:
     name: "Latest PyTorch + DeepSpeed"
@@ -97,61 +69,7 @@ jobs:
           build-args: |
             REF=main
           push: true
-          tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }}
-
-  nightly-torch-deepspeed-docker:
-    name: "Nightly PyTorch + DeepSpeed"
-    # Push CI doesn't need this image
-    if: inputs.image_postfix != '-push-ci'
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Check out code
-        uses: actions/checkout@v2
-      -
-        name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-      -
-        name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker/transformers-pytorch-deepspeed-nightly-gpu
-          build-args: |
-            REF=main
-          push: true
-          tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu
-
-  doc-builder:
-    name: "Doc builder"
-    # Push CI doesn't need this image
-    if: inputs.image_postfix != '-push-ci'
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Check out code
-        uses: actions/checkout@v2
-      -
-        name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-      -
-        name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker/transformers-doc-builder
-          push: true
-          tags: huggingface/transformers-doc-builder
+          tags: huggingface/internal-transformers-pytorch-deepspeed-latest-gpu-tokenizers-main
 
   latest-pytorch:
     name: "Latest PyTorch [dev]"
@@ -179,7 +97,7 @@ jobs:
           build-args: |
             REF=main
           push: true
-          tags: huggingface/transformers-pytorch-gpu
+          tags: huggingface/internal-transformers-pytorch-gpu-tokenizers-main
 
   latest-tensorflow:
     name: "Latest TensorFlow [dev]"
@@ -207,4 +125,4 @@ jobs:
           build-args: |
             REF=main
           push: true
-          tags: huggingface/transformers-tensorflow-gpu
+          tags: huggingface/internal-transformers-tensorflow-gpu-tokenizers-main

docker/transformers-all-latest-gpu/Dockerfile

@@ -16,7 +16,7 @@ ARG INTEL_TORCH_EXT='1.11.0'
 ARG CUDA='cu113'
 
 RUN apt update
-RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs
+RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs curl libssl-dev
 RUN git lfs install
 RUN python3 -m pip install --no-cache-dir --upgrade pip
 
@@ -53,3 +53,12 @@ RUN python3 -m pip install --no-cache-dir decord
 # When installing in editable mode, `transformers` is not recognized as a package.
 # this line must be added in order for python to be aware of transformers.
 RUN cd transformers && python3 setup.py develop
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN cargo --help
+RUN python3 -m pip uninstall -y tokenizers
+RUN git clone https://github.com/huggingface/tokenizers
+RUN python3 -m pip install --no-cache-dir setuptools-rust
+RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile

@@ -8,7 +8,7 @@ ARG PYTORCH='1.12.1'
 ARG CUDA='cu113'
 
 RUN apt -y update
-RUN apt install -y libaio-dev
+RUN apt install -y libaio-dev curl libssl-dev
 RUN python3 -m pip install --no-cache-dir --upgrade pip
 
 ARG REF=main
@@ -33,3 +33,12 @@ RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 py
 RUN cd transformers && python3 setup.py develop
 
 RUN python3 -c "from deepspeed.launcher.runner import main"
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN cargo --help
+RUN python3 -m pip uninstall -y tokenizers
+RUN git clone https://github.com/huggingface/tokenizers
+RUN python3 -m pip install --no-cache-dir setuptools-rust
+RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

docker/transformers-pytorch-gpu/Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face"
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update
-RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg
+RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg curl libssl-dev
 RUN python3 -m pip install --no-cache-dir --upgrade pip
 
 ARG REF=main
@@ -29,3 +29,12 @@ RUN python3 -m pip install -U "itsdangerous<2.1.0"
 # When installing in editable mode, `transformers` is not recognized as a package.
 # this line must be added in order for python to be aware of transformers.
 RUN cd transformers && python3 setup.py develop
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN cargo --help
+RUN python3 -m pip uninstall -y tokenizers
+RUN git clone https://github.com/huggingface/tokenizers
+RUN python3 -m pip install --no-cache-dir setuptools-rust
+RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

docker/transformers-tensorflow-gpu/Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face"
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update
-RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg
+RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg curl libssl-dev
 RUN python3 -m pip install --no-cache-dir --upgrade pip
 
 ARG REF=main
@@ -21,3 +21,12 @@ RUN python3 -m pip install -U "itsdangerous<2.1.0"
 # When installing in editable mode, `transformers` is not recognized as a package.
 # this line must be added in order for python to be aware of transformers.
 RUN cd transformers && python3 setup.py develop
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN cargo --help
+RUN python3 -m pip uninstall -y tokenizers
+RUN git clone https://github.com/huggingface/tokenizers
+RUN python3 -m pip install --no-cache-dir setuptools-rust
+RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python