frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add --jobs 0 for git submodule update (#61311)

Browse Source 
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/61311

Pull Request resolved: https://github.com/pytorch/pytorch/pull/61152

Some related docs about `submodule.fetchJobs`
https://git-scm.com/docs/git-config#Documentation/git-config.txt-submodulefetchJobs

```
time git submodule update --init --recursive
________________________________________________________
Executed in  243.20 secs    fish           external
   usr time   49.64 secs  213.00 micros   49.64 secs
   sys time   29.27 secs  795.00 micros   29.27 secs
```

```
time git submodule update --init --recursive --jobs 4
________________________________________________________
Executed in  143.04 secs    fish           external
   usr time   51.06 secs  246.00 micros   51.06 secs
   sys time   30.96 secs  742.00 micros   30.96 secs
```

```
time git submodule update --init --recursive --jobs 8
________________________________________________________
Executed in  124.64 secs    fish           external
   usr time   51.76 secs  264.00 micros   51.76 secs
   sys time   30.49 secs  739.00 micros   30.49 secs

```

```
time git submodule update --init --recursive --jobs 0 # use all online cpus
 ________________________________________________________
Executed in  129.75 secs    fish           external
   usr time   51.64 secs  181.00 micros   51.64 secs
   sys time   31.49 secs  781.00 micros   31.49 secs

```

Test Plan: Imported from OSS

Reviewed By: 1ntEgr8

Differential Revision: D29560875

Pulled By: zhouzhuojie

fbshipit-source-id: 556027dffe744c66428075a8a1bf64683930aaaf
This commit is contained in:
zhouzhuojie
2021-07-07 15:37:32 -07:00
committed by Facebook GitHub Bot
parent d33066ab3f
commit 6107cf3750
19 changed files with 31 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.azure_pipelines/job_templates/build-verify-publish-template-unix.yml
View File

@ -44,7 +44,7 @@ jobs:
is_official_build: ${{ parameters.is_official_build}}
# Sync and update PyTorch submodules
- bash: git submodule update --init --recursive
- bash: git submodule update --init --recursive --jobs 0
displayName: Update PyTorch submodules
# Build PyTorch and run unit tests - no packaging

2
.azure_pipelines/job_templates/build-verify-publish-template-win.yml
View File

@ -47,7 +47,7 @@ jobs:
is_official_build: ${{ parameters.is_official_build}}
# Sync and update PyTorch submodules
- script: git submodule update --init --recursive
- script: git submodule update --init --recursive --jobs 0
displayName: Update PyTorch submodules
# Build PyTorch and run unit tests - no packaging

8
.circleci/config.yml
View File

@ -496,7 +496,7 @@ jobs:
time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
git submodule sync && git submodule update -q --init --recursive --depth 1
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
@ -1677,7 +1677,7 @@ jobs:
echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}"
time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
git submodule sync && git submodule update -q --init --recursive --depth 1
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace"
export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
@ -1746,7 +1746,7 @@ jobs:
# sync submodules
cd ${PROJ_ROOT}
git submodule sync
git submodule update --init --recursive --depth 1
git submodule update --init --recursive --depth 1 --jobs 0
# export
export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
@ -1839,7 +1839,7 @@ jobs:
echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
git submodule sync && git submodule update -q --init --recursive --depth 1
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace

2
.circleci/scripts/binary_checkout.sh
View File

@ -55,7 +55,7 @@ else
echo "Can't tell what to checkout"
exit 1
fi
retry git submodule update --init --recursive
retry git submodule update --init --recursive --jobs 0
echo "Using Pytorch from "
git --no-pager log --max-count 1
popd

2
.circleci/scripts/binary_ios_build.sh
View File

@ -22,7 +22,7 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
# sync submodules
cd ${PROJ_ROOT}
git submodule sync
git submodule update --init --recursive
git submodule update --init --recursive --jobs 0
# run build script
chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh

6
.circleci/verbatim-sources/job-specs/job-specs-custom.yml
View File

@ -431,7 +431,7 @@
echo "DOCKER_IMAGE: ${DOCKER_IMAGE}:${DOCKER_TAG}"
time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
git submodule sync && git submodule update -q --init --recursive --depth 1
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace"
export id=$(docker run --env-file "${BASH_ENV}" ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
@ -500,7 +500,7 @@
# sync submodules
cd ${PROJ_ROOT}
git submodule sync
git submodule update --init --recursive --depth 1
git submodule update --init --recursive --depth 1 --jobs 0
# export
export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
@ -593,7 +593,7 @@
echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
git submodule sync && git submodule update -q --init --recursive --depth 1
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace

2
.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml
View File

@ -30,7 +30,7 @@ jobs:
time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
git submodule sync && git submodule update -q --init --recursive --depth 1
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace

2
.jenkins/pytorch/macos-lite-interpreter-build-test.sh
View File

@ -4,7 +4,7 @@
source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
git submodule sync --recursive
git submodule update --init --recursive
git submodule update --init --recursive --jobs 0
export CMAKE_PREFIX_PATH=${WORKSPACE_DIR}/miniconda3/
# Build PyTorch

2
.jenkins/pytorch/win-test-helpers/test_custom_backend.bat
View File

@ -1,6 +1,6 @@
call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
git submodule update --init --recursive third_party/pybind11
git submodule update --init --recursive --jobs 0 third_party/pybind11
cd test\custom_backend
:: Build the custom backend library.

2
.jenkins/pytorch/win-test-helpers/test_custom_script_ops.bat
View File

@ -1,6 +1,6 @@
call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
git submodule update --init --recursive third_party/pybind11
git submodule update --init --recursive --jobs 0 third_party/pybind11
cd test\custom_operator
:: Build the custom operator library.

6
CONTRIBUTING.md
View File

@ -102,7 +102,7 @@ cd pytorch
```bash
git pull --rebase
git submodule sync --recursive
git submodule update --init --recursive
git submodule update --init --recursive --jobs 0
```
If you want to have no-op incremental rebuilds (which are fast), see the section below titled "Make no-op build fast."
@ -165,7 +165,7 @@ with `brew install cmake` if you are developing on MacOS or Linux system.
git submodule deinit -f .
git clean -xdf
python setup.py clean
git submodule update --init --recursive # very important to sync the submodules
git submodule update --init --recursive --jobs 0 # very important to sync the submodules
python setup.py develop # then try running the command again
```
4. The main step within `python setup.py develop` is running `make` from the `build` directory. If you want to
@ -173,7 +173,7 @@ with `brew install cmake` if you are developing on MacOS or Linux system.
```bash
ENV_KEY1=ENV_VAL1[, ENV_KEY2=ENV_VAL2]* python setup.py develop
```
* If you run into issue running `git submodule update --init --recursive`. Please try the following:
* If you run into issue running `git submodule update --init --recursive --jobs 0`. Please try the following:
- If you encountered error such as
```
error: Submodule 'third_party/pybind11' could not be updated

2
Dockerfile
View File

@ -38,7 +38,7 @@ RUN curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Mini
FROM dev-base as submodule-update
WORKDIR /opt/pytorch
COPY . .
RUN git submodule update --init --recursive
RUN git submodule update --init --recursive --jobs 0
FROM conda as build
WORKDIR /opt/pytorch

2
README.md
View File

@ -223,7 +223,7 @@ git clone --recursive https://github.com/pytorch/pytorch
cd pytorch
# if you are updating an existing checkout
git submodule sync
git submodule update --init --recursive
git submodule update --init --recursive --jobs 0
```
#### Install PyTorch

2
android/README.md
View File

@ -49,7 +49,7 @@ For this you can use `./scripts/build_pytorch_android.sh` script.
```
git clone https://github.com/pytorch/pytorch.git
cd pytorch
git submodule update --init --recursive
git submodule update --init --recursive --jobs 0
sh ./scripts/build_pytorch_android.sh
```

8
aten/src/ATen/native/quantized/cpu/qnnpack/README.md
View File

@ -70,7 +70,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
cd pytorch
# Optional: update QNNPACK submodule to latest revision
git submodule update --remote third_party/QNNPACK
git submodule update --remote --jobs 0 third_party/QNNPACK
# Build Caffe2 (including binaries) for the host system
# Use only 1 thread for build to avoid out-of-memory failures
@ -97,7 +97,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
cd pytorch
# Optional: update QNNPACK submodule to latest revision
git submodule update --remote third_party/QNNPACK
git submodule update --remote --jobs 0 third_party/QNNPACK
# Build Caffe2 (including binaries) for Android, and push to device
scripts/build_android.sh -DANDROID_TOOLCHAIN=clang -DBUILD_BINARY=ON
@ -127,7 +127,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
cd pytorch
# Optional: update QNNPACK submodule to latest revision
git submodule update --remote third_party/QNNPACK
git submodule update --remote --jobs 0 third_party/QNNPACK
# Build Caffe2 (including binaries) for Android, and push to device
scripts/build_android.sh -DANDROID_ABI=arm64-v8a -DANDROID_TOOLCHAIN=clang -DBUILD_BINARY=ON
@ -164,7 +164,7 @@ git clone --recursive https://github.com/pytorch/pytorch.git
cd pytorch
# Optional: update QNNPACK submodule to latest revision
git submodule update --remote third_party/QNNPACK
git submodule update --remote --jobs 0 third_party/QNNPACK
# Clone PEP repo
cd ~/Code

2
cmake/Modules/FindMKLDNN.cmake
View File

@ -24,7 +24,7 @@ FIND_PACKAGE(BLAS)
FIND_PATH(IDEEP_INCLUDE_DIR ideep.hpp PATHS ${IDEEP_ROOT} PATH_SUFFIXES include)
FIND_PATH(MKLDNN_INCLUDE_DIR mkldnn.hpp mkldnn.h PATHS ${MKLDNN_ROOT} PATH_SUFFIXES include)
IF (NOT MKLDNN_INCLUDE_DIR)
EXECUTE_PROCESS(COMMAND git${CMAKE_EXECUTABLE_SUFFIX} submodule update --init mkl-dnn WORKING_DIRECTORY ${IDEEP_ROOT})
EXECUTE_PROCESS(COMMAND git${CMAKE_EXECUTABLE_SUFFIX} submodule update --init --jobs 0 mkl-dnn WORKING_DIRECTORY ${IDEEP_ROOT})
FIND_PATH(MKLDNN_INCLUDE_DIR mkldnn.hpp mkldnn.h PATHS ${MKLDNN_ROOT} PATH_SUFFIXES include)
ENDIF(NOT MKLDNN_INCLUDE_DIR)

2
docker/cpu-blis/Dockerfile
View File

@ -59,7 +59,7 @@ RUN git clone ${BLIS_URL} && cd blis && \
# Build PyTorch with DNNL+BLIS
RUN git clone https://github.com/pytorch/pytorch.git && cd pytorch && \
git submodule update --init --recursive && \
git submodule update --init --recursive --jobs 0 && \
export PATH=/root/BLISBuild/include/blis:$PATH LD_LIBRARY_PATH=/root/BLISBuild/lib:$LD_LIBRARY_PATH && \
export BLIS_HOME=/root/BLISBuild BLAS=BLIS USE_MKLDNN_CBLAS=ON WITH_BLAS=blis && python setup.py install

4
setup.py
View File

@ -317,7 +317,7 @@ def check_submodules():
def check_for_files(folder, files):
if not any(os.path.exists(os.path.join(folder, f)) for f in files):
report("Could not find any of {} in {}".format(", ".join(files), folder))
report("Did you run 'git submodule update --init --recursive'?")
report("Did you run 'git submodule update --init --recursive --jobs 0'?")
sys.exit(1)
def not_exists_or_empty(folder):
@ -336,7 +336,7 @@ def check_submodules():
print(' --- Submodule initialization took {:.2f} sec'.format(end - start))
except Exception:
print(' --- Submodule initalization failed')
print('Please run:\n\tgit submodule update --init --recursive')
print('Please run:\n\tgit submodule update --init --recursive --jobs 0')
sys.exit(1)
for folder in folders:
check_for_files(folder, ["CMakeLists.txt", "Makefile", "setup.py", "LICENSE", "LICENSE.txt"])

2
torch/utils/benchmark/examples/prepare_e2e.sh
View File

@ -29,7 +29,7 @@ make_clean_env pr_39967
make_clean_env ref_39744
make_clean_env pr_39744
git submodule update --init --recursive
git submodule update --init --recursive --jobs 0
wget https://github.com/pytorch/pytorch/pull/39850.diff
wget https://github.com/pytorch/pytorch/pull/39744.diff