Compiling most of the project

Now compiles all CPU + GPU code, tests + binaries with deps
2025-10-20 21:14:14 +08:00 · 2016-12-08 08:40:29 -05:00
parent 497659ce0d
commit 4255ee9944
2 changed files with 85 additions and 39 deletions
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@ -1,40 +1,67 @@
 # Excludes
 set(Caffe2_DIR_EXCLUDES
-        "/binaries"
-        "/contrib"
-        "/distributed"
-        "/mpi"
-        "/experiments"
-        "/python"
+        "/binaries"      # handled below
+        "/contrib"       # To be handled
+        "/distributed"   # Exclude for now
+        "/mpi"           # Exclude for now
+        "/experiments"   # Exclude for now
+        "/python"        # handled below
+        "/cuda_rtc"      # TODO: Handle this somehow
 )
 set(Caffe2_FILE_EXCLUDES
-        "/core/init_omp.cc"
        "/operators/fully_connected_op_sparse.cc"
        "/test/caffe2_gtest_main.cc"
-        "/core/init_omp.cc"
        "/db/db_test.cc"
-        "/db/leveldb.cc"
-        "/db/lmdb.cc"
        "/db/rocksdb.cc"
        "/db/zmqdb.cc"
+        "/utils/mkl_utils_test.cc"
+        "/mpi/mpi_test.cc"
 )

-set(Caffe2_CPU_FILE_EXCLUDES
-        "/contrib/nervana/nervana_fc_op_gpu_test.cc"
-        "/core/blob_gpu_test.cc"
-        "/core/context_gpu_test.cc"
-        "/mpi/mpi_gpu_test.cc"
-        "/operators/conv_op_cache_cudnn_test.cc"
-        "/operators/elementwise_op_gpu_test.cc"
-        "/operators/fully_connected_op_gpu_test.cc"
-        "/operators/operator_fallback_gpu_test.cc"
-        "/operators/utility_ops_gpu_test.cc"
+set(Caffe2_CPU_TEST_SRCS
+    "/test/caffe2_gtest_main.cc"
+    "/core/parallel_net_test.cc"
+    "/core/timer_test.cc"
+    "/core/workspace_test.cc"
+    "/core/registry_test.cc"
+    "/core/context_test.cc"
+    "/core/init_test.cc"
+    "/core/operator_test.cc"
+    "/core/typeid_test.cc"
+    "/core/predictor_test.cc"
+    "/core/operator_schema_test.cc"
+    "/core/logging_test.cc"
+    "/core/blob_test.cc"
+    "/core/net_test.cc"
+    "/db/db_test.cc"
+    "/operators/utility_ops_test.cc"
+    "/operators/elementwise_op_test.cc"
+    "/operators/fully_connected_op_test.cc"
+    "/operators/conv_op_cache_cudnn_test.cc"
+    "/operators/text_file_reader_utils_test.cc"
+    "/utils/proto_utils_test.cc"
+    "/utils/math_test.cc"
+    "/utils/simple_queue_test.cc"
+    "/utils/mkl_utils_test.cc"
+    "/utils/fixed_divisor_test.cc"
+)
+
+set(Caffe2_GPU_TEST_SRCS
+    "/contrib/nervana/nervana_fc_op_gpu_test.cc"
+    "/core/blob_gpu_test.cc"
+    "/mpi/mpi_gpu_test.cc"
+    "/core/context_gpu_test.cc"
+    "/operators/operator_fallback_gpu_test.cc"
+    "/operators/utility_ops_gpu_test.cc"
+    "/operators/fully_connected_op_gpu_test.cc"
+    "/operators/elementwise_op_gpu_test.cc"
 )

 # Expand all the exludes.
 prepend(Caffe2_DIR_EXCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${Caffe2_DIR_EXCLUDES})
 prepend(Caffe2_FILE_EXCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${Caffe2_FILE_EXCLUDES})
-prepend(Caffe2_CPU_FILE_EXCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${Caffe2_CPU_FILE_EXCLUDES})
+prepend(Caffe2_CPU_TEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR} ${Caffe2_CPU_TEST_SRCS})
+prepend(Caffe2_GPU_TEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR} ${Caffe2_GPU_TEST_SRCS})

 # Get all source directories.
 # TODO(bwasti): GLOB is not considered best practice.
@ -45,16 +72,24 @@ exclude(Caffe2_DIRS "${Caffe2_DIRS}" ${Caffe2_DIR_EXCLUDES})

 # Get all source files from remaining directories.

-# GPU files
+# ---[ GPU files
+# ------[ cuDNN
 foreach(dir ${Caffe2_DIRS})
        file(GLOB tmp ${dir}/*_cudnn.cc)
        set(Caffe2_GPU_SRCS ${Caffe2_GPU_SRCS} ${tmp})
 endforeach()
+# ------[ general GPU
 foreach(dir ${Caffe2_DIRS})
        file(GLOB tmp ${dir}/*_gpu.cc)
        set(Caffe2_GPU_SRCS ${Caffe2_GPU_SRCS} ${tmp})
 endforeach()
+# ------[ CUDA sources
+foreach(dir ${Caffe2_DIRS})
+        file(GLOB tmp ${dir}/*.cu)
+        set(Caffe2_GPU_SRCS ${Caffe2_GPU_SRCS} ${tmp})
+endforeach()
 exclude(Caffe2_GPU_SRCS "${Caffe2_GPU_SRCS}" ${Caffe2_FILE_EXCLUDES})
+exclude(Caffe2_GPU_SRCS "${Caffe2_GPU_SRCS}" ${Caffe2_GPU_TEST_SRCS})

 # CPU files.
 foreach(dir ${Caffe2_DIRS})
@ -62,21 +97,17 @@ foreach(dir ${Caffe2_DIRS})
        set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} ${tmp})
 endforeach()
 exclude(Caffe2_CPU_SRCS "${Caffe2_CPU_SRCS}" ${Caffe2_FILE_EXCLUDES})
-exclude(Caffe2_CPU_SRCS "${Caffe2_CPU_SRCS}" ${Caffe2_CPU_FILE_EXCLUDES})
 exclude(Caffe2_CPU_SRCS "${Caffe2_CPU_SRCS}" ${Caffe2_GPU_SRCS})
+exclude(Caffe2_CPU_SRCS "${Caffe2_CPU_SRCS}" ${Caffe2_CPU_TEST_SRCS})
+exclude(Caffe2_CPU_SRCS "${Caffe2_CPU_SRCS}" ${Caffe2_GPU_TEST_SRCS})

-# CPU test files.
-set(Caffe2_CPU_TEST_SRCS
-        "/test/caffe2_gtest_main.cc"
-)
-prepend(Caffe2_CPU_TEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR} ${Caffe2_CPU_TEST_SRCS})
-
+# exclude GPU files
 # Compile protobufs.
 add_subdirectory(proto)
 include_directories(BEFORE ${CMAKE_BINARY_DIR})

 # Compile exposed libraries.
-LIST(APPEND CMAKE_CXX_FLAGS "--std=c++11 -fPIC")
+LIST(APPEND CMAKE_CXX_FLAGS "-std=c++11 -fPIC")
 add_library(Caffe2_CPU ${Caffe2_CPU_SRCS})
 target_link_libraries(Caffe2_CPU Caffe2_PROTO)

@ -84,15 +115,17 @@ target_link_libraries(Caffe2_CPU Caffe2_PROTO)
 # TODO(slayton): Move this somewhere sane
 find_package(CUDA)
 LIST(APPEND CUDA_NVCC_FLAGS -Xcompiler -std=c++11)
-LIST(APPEND CUDA_NVCC_FLAGS -gencode arch=compute_52,compute=sm_52)
+LIST(APPEND CUDA_NVCC_FLAGS -gencode arch=compute_52,code=sm_52)
 CUDA_ADD_LIBRARY(Caffe2_GPU ${Caffe2_GPU_SRCS})
-target_link_libraries(Caffe2_GPU Caffe2_PROTO glog gflags )
+target_link_libraries(Caffe2_GPU Caffe2_CPU Caffe2_PROTO glog gflags atlas cblas)

 # Compile test binaries.
-add_executable(Caffe2_CPU_TEST ${Caffe2_CPU_TEST_SRCS})
-# TODO: Make BLAS backend interchangeable
-target_link_libraries(Caffe2_CPU_TEST Caffe2_CPU gtest glog atlas cblas ${Caffe2_LINKER_LIBS})
-target_compile_features(Caffe2_CPU_TEST PRIVATE cxx_range_for)
+foreach(test_src ${Caffe2_CPU_TEST_SRCS})
+  get_filename_component(test_name ${test_src} NAME_WE)
+  add_executable(${test_name} "${CMAKE_CURRENT_SOURCE_DIR}/"${test_src})
+  target_link_libraries(${test_name} Caffe2_CPU Caffe2_GPU Caffe_PROTO Caffe2_PROTO glog atlas cblas gtest gtest_main ${Caffe2_LINKER_LIBS})
+  target_compile_features(${test_name} PRIVATE cxx_range_for)
+endforeach()

 # ---[ Python
 add_subdirectory(python)
@ -124,10 +157,8 @@ endif()

 foreach(binary_src ${Caffe2_BINARY_SRCS})
  get_filename_component(bin_name ${binary_src} NAME_WE)
-  # message(STATUS ${bin_name})
-  # message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}/"${binary_src})
  add_executable(${bin_name} "${CMAKE_CURRENT_SOURCE_DIR}/"${binary_src})
-  target_link_libraries(${bin_name} Caffe2_CPU Caffe2_GPU Caffe_PROTO Caffe2_PROTO glog atlas cblas ${Caffe2_LINKER_LIBS})
+  target_link_libraries(${bin_name} -Wl,--whole-archive Caffe2_CPU Caffe2_GPU Caffe_PROTO Caffe2_PROTO glog atlas cblas cnmem ${Caffe2_LINKER_LIBS} -Wl,--no-whole-archive)
 endforeach()


--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@ -101,3 +101,18 @@ if(USE_MPI)
    set(CMAKE_EXE_LINKER_FLAGS ${MPI_CXX_LINK_FLAGS})
  endif()
 endif()
+
+# ---[ OpenMP
+find_package(OpenMP)
+if(OpenMP_FOUND)
+  # set(CMAKE_CXX_FLAGS ${OpenMP_CXX_FLAGS})
+  list(APPEND Caffe2_LINKER_LIBS ${OpenMP_CXX_FLAGS})
+endif()
+
+# ---[ CUB
+include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/third_party/cub)
+
+# ---[ CNMEM
+add_subdirectory(${CMAKE_SOURCE_DIR}/third_party/cnmem)
+include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/third_party/cnmem/include)
+list(APPEND ${Caffe2_LINKER_LIBS} ${CMAKE_SOURCE_DIR}/third_party/cnmem/libcnmem.so)