CMake support for Gloo dependency

Summary:
This also requires a change to cmake/External/nccl.cmake to use the
static NCCL binary instead of the shared object. When the Caffe2/Gloo
build uses the bundled NCCL version it should be packaged up in the
resulting libraries and not cause another runtime dependency on a
library that has to be installed separately.
Closes https://github.com/caffe2/caffe2/pull/218

Differential Revision: D4769926

Pulled By: pietern

fbshipit-source-id: 5c85559992c200d874f4218724823815ffb5adb5

.gitmodules

@@ -34,3 +34,6 @@
 [submodule "third_party/NNPACK"]
 	path = third_party/NNPACK
 	url = https://github.com/Maratyszcza/NNPACK.git
+[submodule "third_party/gloo"]
+	path = third_party/gloo
+	url = https://github.com/facebookincubator/gloo

CMakeLists.txt

@@ -51,6 +51,7 @@ option(USE_ZMQ "Use ZMQ" OFF)
 option(USE_ROCKSDB "Use RocksDB" ON)
 option(USE_REDIS "Use Redis" OFF)
 option(USE_MPI "Use MPI" ON)
+option(USE_GLOO "Use Gloo" ON)
 option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON)
 option(USE_OPENMP "Use OpenMP for parallel code" ON)
 option(BUILD_PYTHON "Build python binaries" ON)

caffe2/contrib/CMakeLists.txt

@@ -1,3 +1,4 @@
+add_subdirectory(gloo)
 add_subdirectory(nccl)
 add_subdirectory(nnpack)
 

caffe2/contrib/gloo/CMakeLists.txt

@@ -0,0 +1,18 @@
+if(USE_GLOO)
+  set(Caffe2_CONTRIB_GLOO_CPU_SRC
+    "${CMAKE_CURRENT_SOURCE_DIR}/allreduce_ops.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/broadcast_ops.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/common_world_ops.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/context.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/store_handler.cc"
+    )
+
+  set(Caffe2_CONTRIB_GLOO_GPU_SRC
+    "${CMAKE_CURRENT_SOURCE_DIR}/allreduce_ops_gpu.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/broadcast_ops_gpu.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/common_world_ops_gpu.cc"
+    )
+
+  set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} ${Caffe2_CONTRIB_GLOO_CPU_SRC} PARENT_SCOPE)
+  set(Caffe2_GPU_SRCS ${Caffe2_GPU_SRCS} ${Caffe2_CONTRIB_GLOO_GPU_SRC} PARENT_SCOPE)
+endif()

cmake/Dependencies.cmake

@@ -292,3 +292,30 @@ if(USE_CNMEM)
     add_definitions(-DCAFFE2_USE_CNMEM)
   endif()
 endif()
+
+if(USE_GLOO)
+  if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+    message(WARNING "Gloo can only be used on Linux.")
+    set(USE_GLOO OFF)
+  else()
+    set(GLOO_INSTALL OFF CACHE BOOL "" FORCE)
+    set(GLOO_STATIC_OR_SHARED STATIC CACHE STRING "" FORCE)
+
+    # Temporarily override variables to avoid building Gloo tests/benchmarks
+    set(__BUILD_TEST ${BUILD_TEST})
+    set(__BUILD_BENCHMARK ${BUILD_BENCHMARK})
+    set(BUILD_TEST OFF)
+    set(BUILD_BENCHMARK OFF)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/gloo)
+    include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third_party/gloo)
+    set(BUILD_TEST ${__BUILD_TEST})
+    set(BUILD_BENCHMARK ${__BUILD_BENCHMARK})
+
+    # Pick the right dependency depending on USE_CUDA
+    if(NOT USE_CUDA)
+      list(APPEND Caffe2_DEPENDENCY_LIBS gloo)
+    else()
+      list(APPEND Caffe2_DEPENDENCY_LIBS gloo_cuda)
+    endif()
+  endif()
+endif()

cmake/External/nccl.cmake

@@ -28,7 +28,7 @@ if (NOT __NCCL_INCLUDED)
 
     set(NCCL_FOUND TRUE)
     set(NCCL_INCLUDE_DIRS ${nccl_PREFIX}/build/include)
-    set(NCCL_LIBRARIES ${nccl_PREFIX}/build/lib/libnccl.so)
+    set(NCCL_LIBRARIES ${nccl_PREFIX}/build/lib/libnccl_static.a)
     set(NCCL_LIBRARY_DIRS ${nccl_PREFIX}/build/lib)
     set(NCCL_EXTERNAL TRUE)
 

cmake/Summary.cmake

@@ -1,4 +1,3 @@
-
 # Prints accumulated Caffe2 configuration summary
 function (Caffe2_print_configuration_summary)
 
@@ -76,5 +75,6 @@ function (Caffe2_print_configuration_summary)
   message(STATUS "  USE_NNPACK            : ${USE_NNPACK}")
   message(STATUS "  USE_OPENMP            : ${USE_OPENMP}")
   message(STATUS "  USE_REDIS             : ${USE_REDIS}")
+  message(STATUS "  USE_GLOO              : ${USE_GLOO}")
 
 endfunction()