Adding gcc4 conda builds (#2283)

* Changes without centos changes

* Changes for protobuf 3.5 and gcc 4.8

* Changing 3.4.1 back to 3.5.1

* Preventing installing two versions of setuptools

* Fixing setuptools bug

.jenkins/build.sh

@@ -60,17 +60,6 @@ if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
   export SKIP_CONDA_TESTS=1
   export CONDA_INSTALL_LOCALLY=1
   "${ROOT_DIR}/scripts/build_anaconda.sh" "$@"
-
-  # The tests all need hypothesis, tabulate, and pydot, which aren't included
-  # in the conda packages
-  conda install -y hypothesis tabulate pydot
-
-  # This build will be tested against onnx tests, which needs onnx installed.
-  # Onnx should be built against the same protobuf that Caffe2 uses, which is
-  # only installed in the conda environment when Caffe2 is.
-  # This path comes from install_anaconda.sh which installs Anaconda into the
-  # docker image
-  PROTOBUF_INCDIR=/opt/conda/include pip install "${ROOT_DIR}/third_party/onnx"
   exit 0
 fi
 

.jenkins/test.sh

@@ -2,6 +2,9 @@
 
 set -ex
 
+LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(cd "$LOCAL_DIR"/.. && pwd)
+
 # Figure out which Python to use
 PYTHON="python"
 if [ -n "$BUILD_ENVIRONMENT" ]; then
@@ -21,15 +24,29 @@ if [[ "$BUILD_ENVIRONMENT" == conda* ]]; then
   # docker image
   PYTHON="/opt/conda/bin/python"
   INSTALL_PREFIX="/opt/conda/"
+
+  # Testing requires separate packages
+  if [[ $BUILD_ENVIRONMENT == *gcc4* ]]; then
+    # These are from conda-forge
+    conda install -yc conda-forge hypothesis tabulate pydot networkx==2.0 click pytest scipy
+    # These packages are from the default channels
+    conda install -y opencv=3.1.0=np112py27_1 pil=1.1.7=py27_2
+  else
+    conda install -y hypothesis tabulate pydot
+  fi
+
+  # This build will be tested against onnx tests, which needs onnx installed.
+  # Onnx should be built against the same protobuf that Caffe2 uses, which is
+  # only installed in the conda environment when Caffe2 is.
+  # This path comes from install_anaconda.sh which installs Anaconda into the
+  # docker image
+  PROTOBUF_INCDIR=/opt/conda/include pip install "${ROOT_DIR}/third_party/onnx"
 fi
 
 # Add the site-packages in the caffe2 install prefix to the PYTHONPATH
 SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix=''))")
 INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
 
-LOCAL_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-ROOT_DIR=$(cd "$LOCAL_DIR"/.. && pwd)
-
 # Skip tests in environments where they are not built/applicable
 if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   echo 'Skipping tests'

conda/cuda/build.sh

@@ -20,6 +20,11 @@ set -ex
 
 echo "Installing caffe2 to ${PREFIX}"
 
+# This is needed for build variants (packages with multiple variants in 
+# conda_build_config.yaml) to remove any files that cmake cached, since
+# conda-build uses the same environment for all the build variants
+rm -rf build
+
 PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
 CMAKE_ARGS=()
 

conda/cuda/conda_build_config.yaml

@@ -1,5 +1,5 @@
 protobuf:
-  - 3.4.1
+  - 3.5.1
 pin_run_as_build:
   protobuf:
     min_pin: x.x

conda/cuda/meta.yaml

@@ -26,6 +26,7 @@ requirements:
     - python
     - protobuf {{ protobuf }}
     - six
+    # other packages here
   run:
     - future
     - glog
@@ -37,6 +38,7 @@ requirements:
     - protobuf
     - python
     - six
+    # other packages here
 
 test:
   imports:

conda/no_cuda/build.sh

@@ -22,6 +22,11 @@ set -ex
 
 echo "Installing caffe2 to ${PREFIX}"
 
+# This is needed for build variants (packages with multiple variants in 
+# conda_build_config.yaml) to remove any files that cmake cached, since
+# conda-build uses the same environment for all the build variants
+rm -rf build
+
 PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
 CMAKE_ARGS=()
 

conda/no_cuda/conda_build_config.yaml

@@ -1,3 +1,5 @@
+protobuf:
+  - 3.5.1
 pin_run_as_build:
   protobuf:
     min_pin: x.x

conda/no_cuda/meta.yaml

@@ -44,9 +44,6 @@ requirements:
     - six
     # other packages here
 
-test:
-  imports:
-    - caffe2.python.core
 
 about:
   home: https://caffe2.ai/

docker/jenkins/common/install_anaconda.sh

@@ -6,28 +6,15 @@ set -ex
 export LANG=C.UTF-8
 export LC_ALL=C.UTF-8
 
-# Install needed packages
-# This also needs build-essentials but that should be installed already
-apt-get update --fix-missing
-apt-get install -y wget
+# Pick correct Anaconda package
+CONDA_PKG_NAME="Anaconda${ANACONDA_VERSION}-5.0.1-Linux-x86_64.sh"
+CONDA_PKG_URL="https://repo.continuum.io/archive/${CONDA_PKG_NAME}"
 
 # Install anaconda
 echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh
-case "$ANACONDA_VERSION" in
-  2*)
-    wget https://repo.continuum.io/archive/Anaconda2-5.0.1-Linux-x86_64.sh -O ~/anaconda.sh
-  ;;
-  3*)
-    wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh -O ~/anaconda.sh
-  ;;
-  *)
-    echo "Invalid ANACONDA_VERSION..."
-    echo $ANACONDA_VERSION
-    exit 1
-  ;;
-esac
-/bin/bash ~/anaconda.sh -b -p /opt/conda
-rm ~/anaconda.sh
+curl -LO "$CONDA_PKG_URL"
+/bin/bash "$CONDA_PKG_NAME" -b -p /opt/conda
+rm "$CONDA_PKG_NAME"
 
 export PATH="/opt/conda/bin:$PATH"
 echo 'export PATH=/opt/conda/bin:$PATH' > ~/.bashrc

docker/jenkins/common/install_nccl.sh

@@ -30,15 +30,9 @@ if [ -n "$NCCL_UBUNTU_VER" ]; then
   apt-get install -y wget
   dpkg -i "${NCCL_DEB}"
 
-  if [ ${CUDA_VERSION:0:3} == 9.1 ]; then
-    # For 9.1 because 2.1.2 for 9.1 does not exist, we will stil use 2.1.4.
-    NCCL_LIB_VERSION="2.1.4-1+cuda${CUDA_VERSION:0:3}"
-  else
-    # Actually installing takes into account CUDA version.
-    # Version 2.1.4 exports symbols it shouldn't, causing a ton of tests to fail.
-    # We pin this to 2.1.2 until this is solved and NVIDIA releases a new version.
-    NCCL_LIB_VERSION="2.1.2-1+cuda${CUDA_VERSION:0:3}"
-  fi
+  # On March 8, 2018 Nvidia began recommending version 2.1.15
+  NCCL_LIB_VERSION="2.1.15-1+cuda${CUDA_VERSION:0:3}"
+
   apt update
-  apt install libnccl2=$NCCL_LIB_VERSION libnccl-dev=$NCCL_LIB_VERSION
+  apt install -y libnccl2=$NCCL_LIB_VERSION libnccl-dev=$NCCL_LIB_VERSION
 fi

docker/jenkins/common/install_python.sh

@@ -122,10 +122,12 @@ pushd pip-9.0.1
 popd
 rm -rf pip-9.0.1*
 
-# Upgrade setuptools
-# setuptools 38.5.2 seems to be buggy, see error in
-# https://ci.pytorch.org/jenkins/job/caffe2-docker/job/py3.6-gcc5-ubuntu16.04/35/consoleFull
-pip install -U pip setuptools!=38.5.2
+if [ -z "${INSTALL_SETUPTOOLS}" ]; then
+  # Upgrade setuptools
+  # setuptools 38.5.2 seems to be buggy, see error in
+  # https://ci.pytorch.org/jenkins/job/caffe2-docker/job/py3.6-gcc5-ubuntu16.04/35/consoleFull
+  pip install -U pip setuptools!=38.5.2
+fi
 
 # tornado 5.0 requires Python 2.7.9+ or 3.4+
 if [[ $($PYTHON -c 'import sys; print(int(sys.version_info <= (2, 7, 9) or sys.version_info <= (3, 4)))' == 1) ]]; then

scripts/build_anaconda.sh

@@ -2,11 +2,15 @@
 
 # NOTE: All parameters to this function are forwared directly to conda-build
 # and so will never be seen by the build.sh
+# TODO change arguments to go to cmake by default
+# TODO handle setting flags in build.sh too
 
 set -ex
 
 # portable_sed: A wrapper around sed that works on both mac and linux, used to
-# alter conda-build files such as the meta.yaml
+# alter conda-build files such as the meta.yaml. It always adds the inplace
+# flag
+#   portable_sed <full regex string> <file>
 portable_sed () {
   if [ "$(uname)" == 'Darwin' ]; then
     sed -i '' "$1" "$2"
@@ -15,6 +19,8 @@ portable_sed () {
   fi
 }
 
+# remove_package: Given a package name, removes any line that mentions that
+# file from the meta.yaml
 remove_package () {
   portable_sed "/$1/d" "${META_YAML}"
 }
@@ -35,8 +41,19 @@ CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
 CONDA_BUILD_ARGS=()
 CMAKE_BUILD_ARGS=()
 
-# Read gcc and Python versions
-# Find which ABI to build for
+# Reinitialize submodules
+git submodule update --init
+
+
+#
+# Read python and gcc version
+#
+
+# Read the gcc version to see what ABI to build for
+if [[ $BUILD_ENVIRONMENT == *gcc4.8* ]]; then
+  GCC_USE_C11=0
+  GCC_VERSION='4.8'
+fi
 if [ "$(uname)" != 'Darwin' -a -z "${GCC_USE_C11}" ]; then
   GCC_VERSION="$(gcc --version | grep --only-matching '[0-9]\.[0-9]\.[0-9]*' | head -1)"
   if [[ "$GCC_VERSION" == 4* ]]; then
@@ -45,6 +62,8 @@ if [ "$(uname)" != 'Darwin' -a -z "${GCC_USE_C11}" ]; then
     GCC_USE_C11=1
   fi
 fi
+
+# Read the python version
 # Specifically 3.6 because the latest Anaconda version is 3.6, and so it's site
 # packages have 3.6 in the name
 PYTHON_VERSION="$(python --version 2>&1 | grep --only-matching '[0-9]\.[0-9]\.[0-9]*')"
@@ -54,10 +73,10 @@ if [[ "$PYTHON_VERSION" == 3.6* ]]; then
   CONDA_BUILD_ARGS+=(" --python 3.6")
 fi
 
-# Reinitialize submodules
-git submodule update --init
 
-# Pick correct conda-build folder
+#
+# Pick the correct conda-build folder
+#
 CAFFE2_CONDA_BUILD_DIR="${CAFFE2_ROOT}/conda"
 if [[ "${BUILD_ENVIRONMENT}" == *full* ]]; then
   CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/cuda_full"
@@ -67,26 +86,34 @@ else
   CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/no_cuda"
 fi
 META_YAML="${CAFFE2_CONDA_BUILD_DIR}/meta.yaml"
+CONDA_BUILD_CONFIG_YAML="${CAFFE2_CONDA_BUILD_DIR}/conda_build_config.yaml"
 
-# Change the package name for CUDA builds to have the specific CUDA and cuDNN
-# version in them
+
+#
+# Build the name of the package depending on CUDA and gcc
+#
 CAFFE2_PACKAGE_NAME="caffe2"
-if [[ "${BUILD_ENVIRONMENT}" == *cuda* ]]; then
-  # Build name of package
-  CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-cuda${CAFFE2_CUDA_VERSION}-cudnn${CAFFE2_CUDNN_VERSION}"
-  if [[ "${BUILD_ENVIRONMENT}" == *full* ]]; then
-    CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-full"
-  fi
-
+if [[ $BUILD_ENVIRONMENT == *cuda* ]]; then
   # CUDA 9.0 and 9.1 are not in conda, and cuDNN is not in conda, so instead of
   # pinning CUDA and cuDNN versions in the conda_build_config and then setting
   # the package name in meta.yaml based off of these values, we let Caffe2
   # take the CUDA and cuDNN versions that it finds in the build environment,
   # and manually set the package name ourself.
-  # WARNING: This does not work on mac.
-  sed -i "s/caffe2-cuda\$/${CAFFE2_PACKAGE_NAME}/" "${META_YAML}"
+  CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-cuda${CAFFE2_CUDA_VERSION}-cudnn${CAFFE2_CUDNN_VERSION}"
 fi
+if [[ -z GCC_USE_C11 ]]; then
+  # gcc compatibility is not tracked by conda-forge, so we track it ourselves
+  CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-gcc${GCC_VERSION:0:3}"
+fi
+if [[ $BUILD_ENVIRONMENT == *full* ]]; then
+  CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-full"
+fi
+portable_sed "s/name: caffe2.*\$/name: ${CAFFE2_PACKAGE_NAME}/" "${META_YAML}"
 
+
+#
+# Handle skipping tests and uploading
+#
 # If skipping tests, remove the test related lines from the meta.yaml and don't
 # upload to Anaconda.org
 if [ -n "$SKIP_CONDA_TESTS" ]; then
@@ -101,13 +128,24 @@ elif [ -n "$UPLOAD_TO_CONDA" ]; then
   CONDA_BUILD_ARGS+=(" --token ${CAFFE2_ANACONDA_ORG_ACCESS_TOKEN}")
 fi
 
+
+#
 # Change flags based on target gcc ABI
+#
 if [[ "$(uname)" != 'Darwin' ]]; then
   if [ "$GCC_USE_C11" -eq 0 ]; then
     CMAKE_BUILD_ARGS+=("-DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0")
     # Default conda channels use gcc 7.2 (for recent packages), conda-forge uses
     # gcc 4.8.5
-    CONDA_BUILD_ARGS+=(" -c conda-forge")
+    # TODO don't do this if user also specified a channel
+    CAFFE2_CONDA_CHANNEL='-c conda-forge'
+
+    # opencv 3.3.1 in conda-forge doesn't have imgcodecs
+    add_package 'opencv' '==3.1.0'
+    if [[ "$PYTHON_VERSION" == 3.* ]]; then
+      # opencv 3.1.0 for python 3 requires numpy 1.12
+      add_package 'numpy' '>1.11'
+    fi
 
   else
     # gflags 2.2.1 is built against the new ABI but gflags 2.2.0 is not
@@ -134,14 +172,21 @@ if [[ "$(uname)" != 'Darwin' ]]; then
     #remove_package 'leveldb'
     #conda install -y 'leveldb=1.20=hf484d3e_1'
   fi
+else
+  # On macOS opencv 3.3.1 (there's only 3.3.1 and 2.4.8) requires protobuf
+  # 3.4
+  portable_sed "s/3.5.1/3.4.1/" "${CONDA_BUILD_CONFIG_YAML}"
 fi
 
+
+#
 # Build Caffe2 with conda-build
+#
 # If --user and --token are set, then this will also upload the built package
 # to Anaconda.org, provided there were no failures and all the tests passed
-CONDA_CMAKE_BUILD_ARGS="$CMAKE_BUILD_ARGS" conda build "${CAFFE2_CONDA_BUILD_DIR}" ${CONDA_BUILD_ARGS[@]} "$@"
+CONDA_CMAKE_BUILD_ARGS="$CMAKE_BUILD_ARGS" conda build "${CAFFE2_CONDA_BUILD_DIR}" $CAFFE2_CONDA_CHANNEL ${CONDA_BUILD_ARGS[@]} "$@"
 
 # Install Caffe2 from the built package into the local conda environment
 if [ -n "$CONDA_INSTALL_LOCALLY" ]; then
-  conda install -y "${CAFFE2_PACKAGE_NAME}" --use-local
+  conda install -y $CAFFE2_CONDA_CHANNEL "${CAFFE2_PACKAGE_NAME}" --use-local
 fi

scripts/read_conda_versions.sh

@@ -1,7 +1,10 @@
 # Simple script used to easily search all packages in conda for their
 # dependency requirements
+# TODO also search through output of ldd
+# TODO update conda info syntax for different channels
 
 if [ -z "$CONDA_ROOT" ]; then
+  # TODO create our own environment
   echo "Please set CONDA_ROOT so that I know where to search for conda libraries"
   echo "I expect CONDA_ROOT to be the path to the current conda environment."
   echo "Also FYI I will probably mess up the current conda environment."