From 6107cf375083ebb6ed538ac3e2c4bacfc5ff9c81 Mon Sep 17 00:00:00 2001 From: zhouzhuojie Date: Wed, 7 Jul 2021 15:37:32 -0700 Subject: [PATCH] Add --jobs 0 for git submodule update (#61311) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/61311 Pull Request resolved: https://github.com/pytorch/pytorch/pull/61152 Some related docs about `submodule.fetchJobs` https://git-scm.com/docs/git-config#Documentation/git-config.txt-submodulefetchJobs ``` time git submodule update --init --recursive ________________________________________________________ Executed in 243.20 secs fish external usr time 49.64 secs 213.00 micros 49.64 secs sys time 29.27 secs 795.00 micros 29.27 secs ``` ``` time git submodule update --init --recursive --jobs 4 ________________________________________________________ Executed in 143.04 secs fish external usr time 51.06 secs 246.00 micros 51.06 secs sys time 30.96 secs 742.00 micros 30.96 secs ``` ``` time git submodule update --init --recursive --jobs 8 ________________________________________________________ Executed in 124.64 secs fish external usr time 51.76 secs 264.00 micros 51.76 secs sys time 30.49 secs 739.00 micros 30.49 secs ``` ``` time git submodule update --init --recursive --jobs 0 # use all online cpus ________________________________________________________ Executed in 129.75 secs fish external usr time 51.64 secs 181.00 micros 51.64 secs sys time 31.49 secs 781.00 micros 31.49 secs ``` Test Plan: Imported from OSS Reviewed By: 1ntEgr8 Differential Revision: D29560875 Pulled By: zhouzhuojie fbshipit-source-id: 556027dffe744c66428075a8a1bf64683930aaaf --- .../job_templates/build-verify-publish-template-unix.yml | 2 +- .../job_templates/build-verify-publish-template-win.yml | 2 +- .circleci/config.yml | 8 ++++---- .circleci/scripts/binary_checkout.sh | 2 +- .circleci/scripts/binary_ios_build.sh | 2 +- .circleci/verbatim-sources/job-specs/job-specs-custom.yml | 6 +++--- .../verbatim-sources/job-specs/pytorch-job-specs.yml | 2 +- .jenkins/pytorch/macos-lite-interpreter-build-test.sh | 2 +- .jenkins/pytorch/win-test-helpers/test_custom_backend.bat | 2 +- .../pytorch/win-test-helpers/test_custom_script_ops.bat | 2 +- CONTRIBUTING.md | 8 ++++---- Dockerfile | 2 +- README.md | 2 +- android/README.md | 2 +- aten/src/ATen/native/quantized/cpu/qnnpack/README.md | 8 ++++---- cmake/Modules/FindMKLDNN.cmake | 2 +- docker/cpu-blis/Dockerfile | 2 +- setup.py | 4 ++-- torch/utils/benchmark/examples/prepare_e2e.sh | 2 +- 19 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.azure_pipelines/job_templates/build-verify-publish-template-unix.yml b/.azure_pipelines/job_templates/build-verify-publish-template-unix.yml index 459d59acc479..6d428c1c6647 100644 --- a/.azure_pipelines/job_templates/build-verify-publish-template-unix.yml +++ b/.azure_pipelines/job_templates/build-verify-publish-template-unix.yml @@ -44,7 +44,7 @@ jobs: is_official_build: ${{ parameters.is_official_build}} # Sync and update PyTorch submodules - - bash: git submodule update --init --recursive + - bash: git submodule update --init --recursive --jobs 0 displayName: Update PyTorch submodules # Build PyTorch and run unit tests - no packaging diff --git a/.azure_pipelines/job_templates/build-verify-publish-template-win.yml b/.azure_pipelines/job_templates/build-verify-publish-template-win.yml index 42fdd32511f3..42f701e1edb9 100644 --- a/.azure_pipelines/job_templates/build-verify-publish-template-win.yml +++ b/.azure_pipelines/job_templates/build-verify-publish-template-win.yml @@ -47,7 +47,7 @@ jobs: is_official_build: ${{ parameters.is_official_build}} # Sync and update PyTorch submodules - - script: git submodule update --init --recursive + - script: git submodule update --init --recursive --jobs 0 displayName: Update PyTorch submodules # Build PyTorch and run unit tests - no packaging diff --git a/.circleci/config.yml b/.circleci/config.yml index a9f7471d96d7..e0f5eed8a03e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -496,7 +496,7 @@ jobs: time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG}) - git submodule sync && git submodule update -q --init --recursive --depth 1 + git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace @@ -1677,7 +1677,7 @@ jobs: echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}" time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null - git submodule sync && git submodule update -q --init --recursive --depth 1 + git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace" export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG}) @@ -1746,7 +1746,7 @@ jobs: # sync submodules cd ${PROJ_ROOT} git submodule sync - git submodule update --init --recursive --depth 1 + git submodule update --init --recursive --depth 1 --jobs 0 # export export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} @@ -1839,7 +1839,7 @@ jobs: echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT" - git submodule sync && git submodule update -q --init --recursive --depth 1 + git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace diff --git a/.circleci/scripts/binary_checkout.sh b/.circleci/scripts/binary_checkout.sh index 17f947d74024..db2b0660d9f5 100755 --- a/.circleci/scripts/binary_checkout.sh +++ b/.circleci/scripts/binary_checkout.sh @@ -55,7 +55,7 @@ else echo "Can't tell what to checkout" exit 1 fi -retry git submodule update --init --recursive +retry git submodule update --init --recursive --jobs 0 echo "Using Pytorch from " git --no-pager log --max-count 1 popd diff --git a/.circleci/scripts/binary_ios_build.sh b/.circleci/scripts/binary_ios_build.sh index bf710dfeba38..f959d40ff898 100644 --- a/.circleci/scripts/binary_ios_build.sh +++ b/.circleci/scripts/binary_ios_build.sh @@ -22,7 +22,7 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} # sync submodules cd ${PROJ_ROOT} git submodule sync -git submodule update --init --recursive +git submodule update --init --recursive --jobs 0 # run build script chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh diff --git a/.circleci/verbatim-sources/job-specs/job-specs-custom.yml b/.circleci/verbatim-sources/job-specs/job-specs-custom.yml index aa7b6e508f3a..80d14fcda2d9 100644 --- a/.circleci/verbatim-sources/job-specs/job-specs-custom.yml +++ b/.circleci/verbatim-sources/job-specs/job-specs-custom.yml @@ -431,7 +431,7 @@ echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}" time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null - git submodule sync && git submodule update -q --init --recursive --depth 1 + git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace" export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG}) @@ -500,7 +500,7 @@ # sync submodules cd ${PROJ_ROOT} git submodule sync - git submodule update --init --recursive --depth 1 + git submodule update --init --recursive --depth 1 --jobs 0 # export export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} @@ -593,7 +593,7 @@ echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT" - git submodule sync && git submodule update -q --init --recursive --depth 1 + git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace diff --git a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml index 64e1dc9824a8..b25e0f3fd0d4 100644 --- a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml +++ b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml @@ -30,7 +30,7 @@ jobs: time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG}) - git submodule sync && git submodule update -q --init --recursive --depth 1 + git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace diff --git a/.jenkins/pytorch/macos-lite-interpreter-build-test.sh b/.jenkins/pytorch/macos-lite-interpreter-build-test.sh index 901f4517ddbd..99c424a5dd6c 100644 --- a/.jenkins/pytorch/macos-lite-interpreter-build-test.sh +++ b/.jenkins/pytorch/macos-lite-interpreter-build-test.sh @@ -4,7 +4,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh" git submodule sync --recursive -git submodule update --init --recursive +git submodule update --init --recursive --jobs 0 export CMAKE_PREFIX_PATH=${WORKSPACE_DIR}/miniconda3/ # Build PyTorch diff --git a/.jenkins/pytorch/win-test-helpers/test_custom_backend.bat b/.jenkins/pytorch/win-test-helpers/test_custom_backend.bat index e209abacca9b..a69a62b23c4e 100644 --- a/.jenkins/pytorch/win-test-helpers/test_custom_backend.bat +++ b/.jenkins/pytorch/win-test-helpers/test_custom_backend.bat @@ -1,6 +1,6 @@ call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat -git submodule update --init --recursive third_party/pybind11 +git submodule update --init --recursive --jobs 0 third_party/pybind11 cd test\custom_backend :: Build the custom backend library. diff --git a/.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat b/.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat index 3dfe30d510c6..50030b2ac234 100644 --- a/.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat +++ b/.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat @@ -1,6 +1,6 @@ call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat -git submodule update --init --recursive third_party/pybind11 +git submodule update --init --recursive --jobs 0 third_party/pybind11 cd test\custom_operator :: Build the custom operator library. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 189cd18099ca..010c21cc4ba7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ cd pytorch ```bash git pull --rebase git submodule sync --recursive -git submodule update --init --recursive +git submodule update --init --recursive --jobs 0 ``` If you want to have no-op incremental rebuilds (which are fast), see the section below titled "Make no-op build fast." @@ -165,15 +165,15 @@ with `brew install cmake` if you are developing on MacOS or Linux system. git submodule deinit -f . git clean -xdf python setup.py clean - git submodule update --init --recursive # very important to sync the submodules - python setup.py develop # then try running the command again + git submodule update --init --recursive --jobs 0 # very important to sync the submodules + python setup.py develop # then try running the command again ``` 4. The main step within `python setup.py develop` is running `make` from the `build` directory. If you want to experiment with some environment variables, you can pass them into the command: ```bash ENV_KEY1=ENV_VAL1[, ENV_KEY2=ENV_VAL2]* python setup.py develop ``` -* If you run into issue running `git submodule update --init --recursive`. Please try the following: +* If you run into issue running `git submodule update --init --recursive --jobs 0`. Please try the following: - If you encountered error such as ``` error: Submodule 'third_party/pybind11' could not be updated diff --git a/Dockerfile b/Dockerfile index cea0fcb9ff04..57c5dae733da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ RUN curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Mini FROM dev-base as submodule-update WORKDIR /opt/pytorch COPY . . -RUN git submodule update --init --recursive +RUN git submodule update --init --recursive --jobs 0 FROM conda as build WORKDIR /opt/pytorch diff --git a/README.md b/README.md index 2d9c7ddacfc2..4f6a3327dafe 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ git clone --recursive https://github.com/pytorch/pytorch cd pytorch # if you are updating an existing checkout git submodule sync -git submodule update --init --recursive +git submodule update --init --recursive --jobs 0 ``` #### Install PyTorch diff --git a/android/README.md b/android/README.md index 7c5b7866bd30..faf69e8720a7 100644 --- a/android/README.md +++ b/android/README.md @@ -49,7 +49,7 @@ For this you can use `./scripts/build_pytorch_android.sh` script. ``` git clone https://github.com/pytorch/pytorch.git cd pytorch -git submodule update --init --recursive +git submodule update --init --recursive --jobs 0 sh ./scripts/build_pytorch_android.sh ``` diff --git a/aten/src/ATen/native/quantized/cpu/qnnpack/README.md b/aten/src/ATen/native/quantized/cpu/qnnpack/README.md index ed6639c4ace9..a7ff4d4ad7fa 100644 --- a/aten/src/ATen/native/quantized/cpu/qnnpack/README.md +++ b/aten/src/ATen/native/quantized/cpu/qnnpack/README.md @@ -70,7 +70,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git cd pytorch # Optional: update QNNPACK submodule to latest revision -git submodule update --remote third_party/QNNPACK +git submodule update --remote --jobs 0 third_party/QNNPACK # Build Caffe2 (including binaries) for the host system # Use only 1 thread for build to avoid out-of-memory failures @@ -97,7 +97,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git cd pytorch # Optional: update QNNPACK submodule to latest revision -git submodule update --remote third_party/QNNPACK +git submodule update --remote --jobs 0 third_party/QNNPACK # Build Caffe2 (including binaries) for Android, and push to device scripts/build_android.sh -DANDROID_TOOLCHAIN=clang -DBUILD_BINARY=ON @@ -127,7 +127,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git cd pytorch # Optional: update QNNPACK submodule to latest revision -git submodule update --remote third_party/QNNPACK +git submodule update --remote --jobs 0 third_party/QNNPACK # Build Caffe2 (including binaries) for Android, and push to device scripts/build_android.sh -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN=clang -DBUILD_BINARY=ON @@ -164,7 +164,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git cd pytorch # Optional: update QNNPACK submodule to latest revision -git submodule update --remote third_party/QNNPACK +git submodule update --remote --jobs 0 third_party/QNNPACK # Clone PEP repo cd ~/Code diff --git a/cmake/Modules/FindMKLDNN.cmake b/cmake/Modules/FindMKLDNN.cmake index c9f787a1721a..a2c67738e782 100644 --- a/cmake/Modules/FindMKLDNN.cmake +++ b/cmake/Modules/FindMKLDNN.cmake @@ -24,7 +24,7 @@ FIND_PACKAGE(BLAS) FIND_PATH(IDEEP_INCLUDE_DIR ideep.hpp PATHS ${IDEEP_ROOT} PATH_SUFFIXES include) FIND_PATH(MKLDNN_INCLUDE_DIR mkldnn.hpp mkldnn.h PATHS ${MKLDNN_ROOT} PATH_SUFFIXES include) IF (NOT MKLDNN_INCLUDE_DIR) - EXECUTE_PROCESS(COMMAND git${CMAKE_EXECUTABLE_SUFFIX} submodule update --init mkl-dnn WORKING_DIRECTORY ${IDEEP_ROOT}) + EXECUTE_PROCESS(COMMAND git${CMAKE_EXECUTABLE_SUFFIX} submodule update --init --jobs 0 mkl-dnn WORKING_DIRECTORY ${IDEEP_ROOT}) FIND_PATH(MKLDNN_INCLUDE_DIR mkldnn.hpp mkldnn.h PATHS ${MKLDNN_ROOT} PATH_SUFFIXES include) ENDIF(NOT MKLDNN_INCLUDE_DIR) diff --git a/docker/cpu-blis/Dockerfile b/docker/cpu-blis/Dockerfile index 1cfda8ec03fc..adfce7e3ad7a 100644 --- a/docker/cpu-blis/Dockerfile +++ b/docker/cpu-blis/Dockerfile @@ -59,7 +59,7 @@ RUN git clone ${BLIS_URL} && cd blis && \ # Build PyTorch with DNNL+BLIS RUN git clone https://github.com/pytorch/pytorch.git && cd pytorch && \ - git submodule update --init --recursive && \ + git submodule update --init --recursive --jobs 0 && \ export PATH=/root/BLISBuild/include/blis:$PATH LD_LIBRARY_PATH=/root/BLISBuild/lib:$LD_LIBRARY_PATH && \ export BLIS_HOME=/root/BLISBuild BLAS=BLIS USE_MKLDNN_CBLAS=ON WITH_BLAS=blis && python setup.py install diff --git a/setup.py b/setup.py index d6dff1cef0ae..5264fe7fa460 100644 --- a/setup.py +++ b/setup.py @@ -317,7 +317,7 @@ def check_submodules(): def check_for_files(folder, files): if not any(os.path.exists(os.path.join(folder, f)) for f in files): report("Could not find any of {} in {}".format(", ".join(files), folder)) - report("Did you run 'git submodule update --init --recursive'?") + report("Did you run 'git submodule update --init --recursive --jobs 0'?") sys.exit(1) def not_exists_or_empty(folder): @@ -336,7 +336,7 @@ def check_submodules(): print(' --- Submodule initialization took {:.2f} sec'.format(end - start)) except Exception: print(' --- Submodule initalization failed') - print('Please run:\n\tgit submodule update --init --recursive') + print('Please run:\n\tgit submodule update --init --recursive --jobs 0') sys.exit(1) for folder in folders: check_for_files(folder, ["CMakeLists.txt", "Makefile", "setup.py", "LICENSE", "LICENSE.txt"]) diff --git a/torch/utils/benchmark/examples/prepare_e2e.sh b/torch/utils/benchmark/examples/prepare_e2e.sh index 342d880ee8ed..06b23ad70ad0 100644 --- a/torch/utils/benchmark/examples/prepare_e2e.sh +++ b/torch/utils/benchmark/examples/prepare_e2e.sh @@ -29,7 +29,7 @@ make_clean_env pr_39967 make_clean_env ref_39744 make_clean_env pr_39744 -git submodule update --init --recursive +git submodule update --init --recursive --jobs 0 wget https://github.com/pytorch/pytorch/pull/39850.diff wget https://github.com/pytorch/pytorch/pull/39744.diff