Install cpp tests when built (#15000)

Summary:
This is broken out of https://github.com/pytorch/pytorch/pull/13733/

We want to install cpp tests so they can ultimately be runnable from that location for Caffe2 tests run from PyTorch builds.

cc pjh5 yf225 anderspapitto
Pull Request resolved: https://github.com/pytorch/pytorch/pull/15000

Reviewed By: pjh5

Differential Revision: D13416253

Pulled By: orionr

fbshipit-source-id: 51280be0a22557a742f90c9f303c58c35cbd4a38

.gitignore

@@ -23,6 +23,7 @@ aten/build/
 aten/src/ATen/Config.h
 aten/src/ATen/cuda/CUDAConfig.h
 build/
+caffe2/cpp_test/
 dist/
 docs/src/**/*
 docs/cpp/build

.jenkins/caffe2/build.sh

@@ -229,6 +229,11 @@ if [[ -z "$INTEGRATED" ]]; then
     exit 1
   fi
 
+  # This is to save test binaries for testing
+  mv "$INSTALL_PREFIX/test/" "$INSTALL_PREFIX/cpp_test/"
+
+  ls $INSTALL_PREFIX
+
 else
 
   # sccache will be stuck if  all cores are used for compiling
@@ -237,10 +242,12 @@ else
     export MAX_JOBS=`expr $(nproc) - 1`
   fi
 
-  USE_LEVELDB=1 USE_LMDB=1 USE_OPENCV=1 BUILD_BINARY=1 python setup.py install --user
+  USE_LEVELDB=1 USE_LMDB=1 USE_OPENCV=1 BUILD_TEST=1 BUILD_BINARY=1 python setup.py install --user
 
   # This is to save test binaries for testing
   cp -r torch/lib/tmp_install $INSTALL_PREFIX
+  mkdir -p "$INSTALL_PREFIX/cpp_test/"
+  cp -r caffe2/test/* "$INSTALL_PREFIX/cpp_test/"
 
   ls $INSTALL_PREFIX
 

.jenkins/caffe2/test.sh

@@ -12,12 +12,18 @@ if [[ "${BUILD_ENVIRONMENT}" =~ py((2|3)\.?[0-9]?\.?[0-9]?) ]]; then
   PYTHON="python${BASH_REMATCH[1]}"
 fi
 
-# The prefix must mirror the setting from build.sh
-INSTALL_PREFIX="/usr/local/caffe2"
-
 # Add the site-packages in the caffe2 install prefix to the PYTHONPATH
 SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix=''))")
-INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
+
+# Find where Caffe2 is installed. This will be the absolute path to the
+# site-packages of the active Python installation
+INSTALL_SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib())")
+INSTALL_PREFIX="$INSTALL_SITE_DIR/caffe2"
+if [ ! -d "$INSTALL_PREFIX/cpp_test" ]; then
+  echo "Directory $INSTALL_PREFIX/cpp_test not found. Fallback to legacy location."
+  INSTALL_PREFIX="/usr/local/caffe2"
+  INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
+fi
 
 # Skip tests in environments where they are not built/applicable
 if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
@@ -46,20 +52,20 @@ echo "Running C++ tests.."
 gtest_reports_dir="${TEST_DIR}/cpp"
 junit_reports_dir="${TEST_DIR}/junit_reports"
 mkdir -p "$gtest_reports_dir" "$junit_reports_dir"
-for test in $(find "${INSTALL_PREFIX}/test" -executable -type f); do
+for test in $(find "${INSTALL_PREFIX}/cpp_test" -executable -type f); do
   case "$test" in
     # skip tests we know are hanging or bad
     */mkl_utils_test|*/aten/integer_divider_test)
       continue
       ;;
     */scalar_tensor_test|*/basic|*/native_test)
-	  if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+      if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
-		continue
+        continue
-	  else
+      else
-	    "$test"
+        "$test"
-	  fi
+      fi
-	  ;;
+      ;;
-	*)
+    *)
       # Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
       # planning to migrate to gtest as the common PyTorch c++ test suite, we
       # currently do NOT use the xml test reporter, because Catch doesn't

CMakeLists.txt

@@ -75,7 +75,7 @@ cmake_dependent_option(
     "NOT BUILD_SHARED_LIBS" OFF)
 option(BUILD_TEST "Build C++ test binaries (need gtest and gbenchmark)" OFF)
 cmake_dependent_option(
-    INSTALL_TEST "Install test binaries if BUILD_TEST is on" OFF
+    INSTALL_TEST "Install test binaries if BUILD_TEST is on" ON
     "BUILD_TEST" OFF)
 option(USE_ACL "Use ARM Compute Library" OFF)
 option(USE_ASAN "Use Address Sanitizer" OFF)

setup.py

@@ -1025,7 +1025,7 @@ if __name__ == '__main__':
                 'share/cmake/Torch/*.cmake',
             ],
             'caffe2': [
-                rel_site_packages + '/caffe2/**/*.py'
+                'cpp_test/*',
             ]
         },
     )

tools/build_pytorch_libs.sh

@@ -342,4 +342,23 @@ $SYNC_COMMAND -r "$INSTALL_DIR/include" .
 if [ -d "$INSTALL_DIR/bin/" ]; then
     $SYNC_COMMAND -r "$INSTALL_DIR/bin/"/* .
 fi
+
+# Copy the test files to pytorch/caffe2 manually
+# They were built in pytorch/torch/lib/tmp_install/test
+# Why do we do this? So, setup.py has this section called 'package_data' which
+# you need to specify to include non-default files (usually .py files).
+# package_data takes a map from 'python package' to 'globs of files to
+# include'. By 'python package', it means a folder with an __init__.py file
+# that's not excluded in the find_packages call earlier in setup.py. So to
+# include our cpp_test into the site-packages folder in
+# site-packages/caffe2/cpp_test, we have to copy the cpp_test folder into the
+# root caffe2 folder and then tell setup.py to include them. Having another
+# folder like site-packages/caffe2_cpp_test would also be possible by adding a
+# caffe2_cpp_test folder to pytorch with an __init__.py in it.
+if [[ "$INSTALL_TEST" == "ON" ]]; then
+    echo "Copying $INSTALL_DIR/test to $BASE_DIR/caffe2/cpp_test"
+    mkdir -p "$BASE_DIR/caffe2/cpp_test/"
+    $SYNC_COMMAND -r "$INSTALL_DIR/test/"/* "$BASE_DIR/caffe2/cpp_test/"
+fi
+
 popd > /dev/null