Add --jobs 0 for git submodule update (#61311)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/61311

Pull Request resolved: https://github.com/pytorch/pytorch/pull/61152

Some related docs about `submodule.fetchJobs`
https://git-scm.com/docs/git-config#Documentation/git-config.txt-submodulefetchJobs

```
time git submodule update --init --recursive
________________________________________________________
Executed in  243.20 secs    fish           external
   usr time   49.64 secs  213.00 micros   49.64 secs
   sys time   29.27 secs  795.00 micros   29.27 secs
```

```
time git submodule update --init --recursive --jobs 4
________________________________________________________
Executed in  143.04 secs    fish           external
   usr time   51.06 secs  246.00 micros   51.06 secs
   sys time   30.96 secs  742.00 micros   30.96 secs
```

```
time git submodule update --init --recursive --jobs 8
________________________________________________________
Executed in  124.64 secs    fish           external
   usr time   51.76 secs  264.00 micros   51.76 secs
   sys time   30.49 secs  739.00 micros   30.49 secs

```

```
time git submodule update --init --recursive --jobs 0 # use all online cpus
 ________________________________________________________
Executed in  129.75 secs    fish           external
   usr time   51.64 secs  181.00 micros   51.64 secs
   sys time   31.49 secs  781.00 micros   31.49 secs

```

Test Plan: Imported from OSS

Reviewed By: 1ntEgr8

Differential Revision: D29560875

Pulled By: zhouzhuojie

fbshipit-source-id: 556027dffe744c66428075a8a1bf64683930aaaf

.azure_pipelines/job_templates/build-verify-publish-template-unix.yml

@@ -44,7 +44,7 @@ jobs:
         is_official_build: ${{ parameters.is_official_build}}
 
     # Sync and update PyTorch submodules
-    - bash: git submodule update --init --recursive
+    - bash: git submodule update --init --recursive --jobs 0
       displayName: Update PyTorch submodules
 
     # Build PyTorch and run unit tests - no packaging

.azure_pipelines/job_templates/build-verify-publish-template-win.yml

@@ -47,7 +47,7 @@ jobs:
         is_official_build: ${{ parameters.is_official_build}}
 
     # Sync and update PyTorch submodules
-    - script: git submodule update --init --recursive
+    - script: git submodule update --init --recursive --jobs 0
       displayName: Update PyTorch submodules
 
     # Build PyTorch and run unit tests - no packaging

.circleci/config.yml

@@ -496,7 +496,7 @@ jobs:
           time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
           export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
 
-          git submodule sync && git submodule update -q --init --recursive --depth 1
+          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
 
           docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 
@@ -1677,7 +1677,7 @@ jobs:
           echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}"
           time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
 
-          git submodule sync && git submodule update -q --init --recursive --depth 1
+          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
           VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace"
           export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
 
@@ -1746,7 +1746,7 @@ jobs:
             # sync submodules
             cd ${PROJ_ROOT}
             git submodule sync
-            git submodule update --init --recursive --depth 1
+            git submodule update --init --recursive --depth 1 --jobs 0
 
             # export
             export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
@@ -1839,7 +1839,7 @@ jobs:
 
           echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
 
-          git submodule sync && git submodule update -q --init --recursive --depth 1
+          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
 
           docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 

.circleci/scripts/binary_checkout.sh

@@ -55,7 +55,7 @@ else
   echo "Can't tell what to checkout"
   exit 1
 fi
-retry git submodule update --init --recursive
+retry git submodule update --init --recursive --jobs 0
 echo "Using Pytorch from "
 git --no-pager log --max-count 1
 popd

.circleci/scripts/binary_ios_build.sh

@@ -22,7 +22,7 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
 # sync submodules
 cd ${PROJ_ROOT}
 git submodule sync
-git submodule update --init --recursive
+git submodule update --init --recursive --jobs 0
 
 # run build script
 chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh

.circleci/verbatim-sources/job-specs/job-specs-custom.yml

@@ -431,7 +431,7 @@
           echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}"
           time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
 
-          git submodule sync && git submodule update -q --init --recursive --depth 1
+          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
           VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace"
           export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
 
@@ -500,7 +500,7 @@
             # sync submodules
             cd ${PROJ_ROOT}
             git submodule sync
-            git submodule update --init --recursive --depth 1
+            git submodule update --init --recursive --depth 1 --jobs 0
 
             # export
             export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
@@ -593,7 +593,7 @@
 
           echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
 
-          git submodule sync && git submodule update -q --init --recursive --depth 1
+          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
 
           docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 

.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml

@@ -30,7 +30,7 @@ jobs:
           time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
           export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
 
-          git submodule sync && git submodule update -q --init --recursive --depth 1
+          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
 
           docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 

.jenkins/pytorch/macos-lite-interpreter-build-test.sh

@@ -4,7 +4,7 @@
 source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
 
 git submodule sync --recursive
-git submodule update --init --recursive
+git submodule update --init --recursive --jobs 0
 export CMAKE_PREFIX_PATH=${WORKSPACE_DIR}/miniconda3/
 
 # Build PyTorch

.jenkins/pytorch/win-test-helpers/test_custom_backend.bat

@@ -1,6 +1,6 @@
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
 
-git submodule update --init --recursive third_party/pybind11
+git submodule update --init --recursive --jobs 0 third_party/pybind11
 cd test\custom_backend
 
 :: Build the custom backend library.

.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat

@@ -1,6 +1,6 @@
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
 
-git submodule update --init --recursive third_party/pybind11
+git submodule update --init --recursive --jobs 0 third_party/pybind11
 cd test\custom_operator
 
 :: Build the custom operator library.

CONTRIBUTING.md

@@ -102,7 +102,7 @@ cd pytorch
 ```bash
 git pull --rebase
 git submodule sync --recursive
-git submodule update --init --recursive
+git submodule update --init --recursive --jobs 0
 ```
 
 If you want to have no-op incremental rebuilds (which are fast), see the section below titled "Make no-op build fast."
@@ -165,15 +165,15 @@ with `brew install cmake` if you are developing on MacOS or Linux system.
   git submodule deinit -f .
   git clean -xdf
   python setup.py clean
-  git submodule update --init --recursive # very important to sync the submodules
+  git submodule update --init --recursive --jobs 0 # very important to sync the submodules
-  python setup.py develop                 # then try running the command again
+  python setup.py develop                          # then try running the command again
   ```
   4. The main step within `python setup.py develop` is running `make` from the `build` directory. If you want to
   experiment with some environment variables, you can pass them into the command:
   ```bash
   ENV_KEY1=ENV_VAL1[, ENV_KEY2=ENV_VAL2]* python setup.py develop
   ```
-* If you run into issue running `git submodule update --init --recursive`. Please try the following:
+* If you run into issue running `git submodule update --init --recursive --jobs 0`. Please try the following:
   - If you encountered error such as
     ```
     error: Submodule 'third_party/pybind11' could not be updated

Dockerfile

@@ -38,7 +38,7 @@ RUN curl -fsSL -v -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Mini
 FROM dev-base as submodule-update
 WORKDIR /opt/pytorch
 COPY . .
-RUN git submodule update --init --recursive
+RUN git submodule update --init --recursive --jobs 0
 
 FROM conda as build
 WORKDIR /opt/pytorch

README.md

@@ -223,7 +223,7 @@ git clone --recursive https://github.com/pytorch/pytorch
 cd pytorch
 # if you are updating an existing checkout
 git submodule sync
-git submodule update --init --recursive
+git submodule update --init --recursive --jobs 0
 ```
 
 #### Install PyTorch

android/README.md

@@ -49,7 +49,7 @@ For this you can use `./scripts/build_pytorch_android.sh` script.
 ```
 git clone https://github.com/pytorch/pytorch.git
 cd pytorch
-git submodule update --init --recursive
+git submodule update --init --recursive --jobs 0
 sh ./scripts/build_pytorch_android.sh
 ```
 

aten/src/ATen/native/quantized/cpu/qnnpack/README.md

@@ -70,7 +70,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
 cd pytorch
 
 # Optional: update QNNPACK submodule to latest revision
-git submodule update --remote third_party/QNNPACK
+git submodule update --remote --jobs 0 third_party/QNNPACK
 
 # Build Caffe2 (including binaries) for the host system
 # Use only 1 thread for build to avoid out-of-memory failures
@@ -97,7 +97,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
 cd pytorch
 
 # Optional: update QNNPACK submodule to latest revision
-git submodule update --remote third_party/QNNPACK
+git submodule update --remote --jobs 0 third_party/QNNPACK
 
 # Build Caffe2 (including binaries) for Android, and push to device
 scripts/build_android.sh -DANDROID_TOOLCHAIN=clang -DBUILD_BINARY=ON
@@ -127,7 +127,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
 cd pytorch
 
 # Optional: update QNNPACK submodule to latest revision
-git submodule update --remote third_party/QNNPACK
+git submodule update --remote --jobs 0 third_party/QNNPACK
 
 # Build Caffe2 (including binaries) for Android, and push to device
 scripts/build_android.sh -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN=clang -DBUILD_BINARY=ON
@@ -164,7 +164,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
 cd pytorch
 
 # Optional: update QNNPACK submodule to latest revision
-git submodule update --remote third_party/QNNPACK
+git submodule update --remote --jobs 0 third_party/QNNPACK
 
 # Clone PEP repo
 cd ~/Code

cmake/Modules/FindMKLDNN.cmake

@@ -24,7 +24,7 @@ FIND_PACKAGE(BLAS)
 FIND_PATH(IDEEP_INCLUDE_DIR ideep.hpp PATHS ${IDEEP_ROOT} PATH_SUFFIXES include)
 FIND_PATH(MKLDNN_INCLUDE_DIR mkldnn.hpp mkldnn.h PATHS ${MKLDNN_ROOT} PATH_SUFFIXES include)
 IF (NOT MKLDNN_INCLUDE_DIR)
-  EXECUTE_PROCESS(COMMAND git${CMAKE_EXECUTABLE_SUFFIX} submodule update --init mkl-dnn WORKING_DIRECTORY ${IDEEP_ROOT})
+  EXECUTE_PROCESS(COMMAND git${CMAKE_EXECUTABLE_SUFFIX} submodule update --init --jobs 0 mkl-dnn WORKING_DIRECTORY ${IDEEP_ROOT})
   FIND_PATH(MKLDNN_INCLUDE_DIR mkldnn.hpp mkldnn.h PATHS ${MKLDNN_ROOT} PATH_SUFFIXES include)
 ENDIF(NOT MKLDNN_INCLUDE_DIR)
 

docker/cpu-blis/Dockerfile

@@ -59,7 +59,7 @@ RUN git clone ${BLIS_URL} && cd blis && \
 
 # Build PyTorch with DNNL+BLIS
 RUN git clone https://github.com/pytorch/pytorch.git && cd pytorch && \
-    git submodule update --init --recursive && \
+    git submodule update --init --recursive --jobs 0 && \
     export PATH=/root/BLISBuild/include/blis:$PATH LD_LIBRARY_PATH=/root/BLISBuild/lib:$LD_LIBRARY_PATH && \
     export BLIS_HOME=/root/BLISBuild BLAS=BLIS USE_MKLDNN_CBLAS=ON WITH_BLAS=blis && python setup.py install
 

setup.py

@@ -317,7 +317,7 @@ def check_submodules():
     def check_for_files(folder, files):
         if not any(os.path.exists(os.path.join(folder, f)) for f in files):
             report("Could not find any of {} in {}".format(", ".join(files), folder))
-            report("Did you run 'git submodule update --init --recursive'?")
+            report("Did you run 'git submodule update --init --recursive --jobs 0'?")
             sys.exit(1)
 
     def not_exists_or_empty(folder):
@@ -336,7 +336,7 @@ def check_submodules():
             print(' --- Submodule initialization took {:.2f} sec'.format(end - start))
         except Exception:
             print(' --- Submodule initalization failed')
-            print('Please run:\n\tgit submodule update --init --recursive')
+            print('Please run:\n\tgit submodule update --init --recursive --jobs 0')
             sys.exit(1)
     for folder in folders:
         check_for_files(folder, ["CMakeLists.txt", "Makefile", "setup.py", "LICENSE", "LICENSE.txt"])

torch/utils/benchmark/examples/prepare_e2e.sh

@@ -29,7 +29,7 @@ make_clean_env pr_39967
 make_clean_env ref_39744
 make_clean_env pr_39744
 
-git submodule update --init --recursive
+git submodule update --init --recursive --jobs 0
 wget https://github.com/pytorch/pytorch/pull/39850.diff
 wget https://github.com/pytorch/pytorch/pull/39744.diff