From a90a3acbeec7deebecf7baf53c8f0fdf7b37e5ae Mon Sep 17 00:00:00 2001 From: Jane Xu Date: Mon, 26 Apr 2021 09:17:12 -0700 Subject: [PATCH] Use JIT Plug-in for coverage to cover JIT'd functions and methods (#56310) Summary: This PR is step 2 (after https://github.com/pytorch/pytorch/issues/56708) to having JIT coverage--it actually uses the plug-in in CI! Disclaimer: note that this will mark the entire JIT'd function/method as covered without seeking proof that the compiled code has been executed. This means that even if the code chunk is merely compiled and not run, it will get marked as covered. Pull Request resolved: https://github.com/pytorch/pytorch/pull/56310 Test Plan: We should see coverage improvements in CI after. A file to look out for would be `torch/jit/quantized.py`, which should have more coverage after this PR, which it does! https://codecov.io/gh/pytorch/pytorch/src/d3283ccd8c898e7a1c5b46179a0b9147c87c53b9/torch/jit/quantized.py vs https://codecov.io/gh/pytorch/pytorch/src/master/torch/jit/quantized.py More generally, the whole jit folder got ~3% increase in coverage, I believe. Reviewed By: walterddr Differential Revision: D28000672 Pulled By: janeyx99 fbshipit-source-id: 6712979d63a5e1224a92ee9bd9679ec62cf1cbba --- .circleci/docker/common/install_conda.sh | 3 ++- .coveragerc | 2 ++ .jenkins/pytorch/test.sh | 1 + .jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat | 3 ++- .jenkins/pytorch/win-test.sh | 3 ++- scripts/onnx/test.sh | 1 + 6 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.circleci/docker/common/install_conda.sh b/.circleci/docker/common/install_conda.sh index 0f445a51addb..3a9fb407335b 100755 --- a/.circleci/docker/common/install_conda.sh +++ b/.circleci/docker/common/install_conda.sh @@ -102,6 +102,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then # TODO: Why is scipy pinned # Pin MyPy version because new errors are likely to appear with each release # Pin hypothesis to avoid flakiness: https://github.com/pytorch/pytorch/issues/31136 + # Pin coverage so we can use COVERAGE_RCFILE as_jenkins pip install --progress-bar off pytest \ scipy==1.1.0 \ scikit-image \ @@ -111,7 +112,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then llvmlite \ unittest-xml-reporting \ boto3==1.16.34 \ - coverage \ + coverage==5.5 \ hypothesis==4.53.2 \ mypy==0.812 \ tb-nightly diff --git a/.coveragerc b/.coveragerc index ad54d65b7ec9..428fe5243a35 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,6 @@ [run] +plugins = + coverage_plugins.jit_plugin omit = */tmp* */Temp/* diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh index b0d768e4c847..3daccd99c391 100755 --- a/.jenkins/pytorch/test.sh +++ b/.jenkins/pytorch/test.sh @@ -26,6 +26,7 @@ fi if [[ "$BUILD_ENVIRONMENT" == *coverage* ]]; then export PYTORCH_COLLECT_COVERAGE=1 export COVERAGE_RCFILE="$PWD/.coveragerc" # coverage config file needed for plug-ins and settings to work + pip install -e tools/coverage_plugins_package # allows coverage to run with JitPlugin for JIT coverage fi if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then diff --git a/.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat b/.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat index 2766bf0e58e6..3eeeb3c5b490 100644 --- a/.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat +++ b/.jenkins/pytorch/win-test-helpers/setup_pytorch_env.bat @@ -43,7 +43,8 @@ pip install "ninja==1.10.0.post1" future "hypothesis==4.53.2" "librosa>=0.6.2" p :: TODO: All sharded configs run coverage. We should change that to be only one config, but right now we will just :: install coverage everywhere. Tracked: https://github.com/pytorch/pytorch/issues/56264 -python -mpip install coverage +python -mpip install coverage==5.5 +python -mpip install -e tools/coverage_plugins_package if %errorlevel% neq 0 ( exit /b %errorlevel% ) diff --git a/.jenkins/pytorch/win-test.sh b/.jenkins/pytorch/win-test.sh index 2ffac5154523..177fc8014ff3 100755 --- a/.jenkins/pytorch/win-test.sh +++ b/.jenkins/pytorch/win-test.sh @@ -92,7 +92,8 @@ echo "TEST PASSED" if [[ "${BUILD_ENVIRONMENT}" == "pytorch-win-vs2019-cuda10-cudnn7-py3" ]]; then pushd "$TEST_DIR" - python -mpip install coverage + python -mpip install coverage==5.5 + python -mpip install -e "$PROJECT_DIR/tools/coverage_plugins_package" echo "Generating XML coverage report" time python -mcoverage xml popd diff --git a/scripts/onnx/test.sh b/scripts/onnx/test.sh index d359efd86270..efc9b31f5cce 100755 --- a/scripts/onnx/test.sh +++ b/scripts/onnx/test.sh @@ -29,6 +29,7 @@ fi pip install pytest scipy hypothesis # these may not be necessary pip install pytest-cov # installing since `coverage run -m pytest ..` doesn't work +pip install -e tools/coverage_plugins_package # allows coverage to run w/o failing due to a missing plug-in # realpath might not be available on MacOS script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")