CMake integration for Int8 ops

Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/13145 Differential Revision: D10860849 Pulled By: Maratyszcza fbshipit-source-id: fdbcc23ff9beaeaedfd561176df6cfe87685c1f5
2025-10-20 21:14:14 +08:00 · 2018-10-25 22:13:18 -07:00
parent 4870b1b68f
commit 5e73b828bd
10 changed files with 130 additions and 2 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -107,6 +107,7 @@ option(USE_OPENCL "Use OpenCL" OFF)
 option(USE_OPENCV "Use OpenCV" ON)
 option(USE_OPENMP "Use OpenMP for parallel code" OFF)
 option(USE_PROF "Use profiling" OFF)
+option(USE_QNNPACK "Use QNNPACK (quantized 8-bit operators)" ON)
 option(USE_REDIS "Use Redis" OFF)
 option(USE_ROCKSDB "Use RocksDB" OFF)
 option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@ -90,6 +90,9 @@ if(NOT BUILD_ATEN_ONLY)
  if (BUILD_CAFFE2_OPS)
    add_subdirectory(operators)
    add_subdirectory(operators/rnn)
+    if (USE_QNNPACK)
+      add_subdirectory(operators/quantized)
+    endif()
  endif()
  add_subdirectory(opt)
  add_subdirectory(perfkernels)
--- a/caffe2/operators/quantized/CMakeLists.txt
+++ b/caffe2/operators/quantized/CMakeLists.txt
@ -0,0 +1,31 @@
+# ---[ CPU files only
+list(APPEND Caffe2_CPU_SRCS
+  "${CMAKE_CURRENT_SOURCE_DIR}/init_qnnpack.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_add_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_average_pool_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_channel_shuffle_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_concat_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_conv_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_conv_transpose_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_dequantize_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_fc_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_flatten_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_given_tensor_fill_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_leaky_relu_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_max_pool_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_quantize_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_relu_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_reshape_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_resize_nearest_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_roi_align_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_slice_op.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_softmax_op.cc")
+
+# ---[ CPU test files
+list(APPEND Caffe2_CPU_TEST_SRCS
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_test.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/int8_roi_align_op_test.cc")
+
+# ---[ Send the lists to the parent scope.
+set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS} PARENT_SCOPE)
+set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} PARENT_SCOPE)
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@ -91,6 +91,73 @@ set(CONFU_DEPENDENCIES_SOURCE_DIR ${PROJECT_BINARY_DIR}/confu-srcs
 set(CONFU_DEPENDENCIES_BINARY_DIR ${PROJECT_BINARY_DIR}/confu-deps
  CACHE PATH "Confu-style dependencies binary directory")

+# ---[ QNNPACK
+if(USE_QNNPACK)
+  if (NOT IOS AND NOT (CMAKE_SYSTEM_NAME MATCHES "^(Android|Linux|Darwin)$"))
+    message(WARNING
+      "Target platform \"${CMAKE_SYSTEM_NAME}\" is not supported in QNNPACK. "
+      "Supported platforms are Android, iOS, Linux, and macOS. "
+      "Turn this warning off by USE_QNNPACK=OFF.")
+    set(USE_QNNPACK OFF)
+  endif()
+  if (NOT IOS AND NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "^(i686|AMD64|x86_64|armv[0-9].*|arm64|aarch64)$"))
+    message(WARNING
+      "Target architecture \"${CMAKE_SYSTEM_PROCESSOR}\" is not supported in QNNPACK. "
+      "Supported platforms are x86, x86-64, ARM, and ARM64. "
+      "Turn this warning off by USE_QNNPACK=OFF.")
+    set(USE_QNNPACK OFF)
+  endif()
+  if (USE_QNNPACK)
+    set(CAFFE2_THIRD_PARTY_ROOT "${PROJECT_SOURCE_DIR}/third_party")
+
+    # Directories for QNNPACK dependencies submoduled in Caffe2
+    if (NOT DEFINED CPUINFO_SOURCE_DIR)
+      set(CPUINFO_SOURCE_DIR "${CAFFE2_THIRD_PARTY_ROOT}/cpuinfo" CACHE STRING "cpuinfo source directory")
+    endif()
+    if (NOT DEFINED QNNPACK_SOURCE_DIR)
+      set(QNNPACK_SOURCE_DIR "${CAFFE2_THIRD_PARTY_ROOT}/QNNPACK" CACHE STRING "QNNPACK source directory")
+    endif()
+    if (NOT DEFINED FP16_SOURCE_DIR)
+      set(FP16_SOURCE_DIR "${CAFFE2_THIRD_PARTY_ROOT}/FP16" CACHE STRING "FP16 source directory")
+    endif()
+    if (NOT DEFINED FXDIV_SOURCE_DIR)
+      set(FXDIV_SOURCE_DIR "${CAFFE2_THIRD_PARTY_ROOT}/FXdiv" CACHE STRING "FXdiv source directory")
+    endif()
+    if (NOT DEFINED PSIMD_SOURCE_DIR)
+      set(PSIMD_SOURCE_DIR "${CAFFE2_THIRD_PARTY_ROOT}/psimd" CACHE STRING "PSimd source directory")
+    endif()
+    if (NOT DEFINED PTHREADPOOL_SOURCE_DIR)
+      set(PTHREADPOOL_SOURCE_DIR "${CAFFE2_THIRD_PARTY_ROOT}/pthreadpool" CACHE STRING "pthreadpool source directory")
+    endif()
+
+    if(NOT TARGET qnnpack)
+      set(QNNPACK_BUILD_TESTS OFF CACHE BOOL "")
+      set(QNNPACK_BUILD_BENCHMARKS OFF CACHE BOOL "")
+      set(QNNPACK_CUSTOM_THREADPOOL ON CACHE BOOL "")
+      set(QNNPACK_LIBRARY_TYPE "static" CACHE STRING "")
+      set(PTHREADPOOL_LIBRARY_TYPE "static" CACHE STRING "")
+      set(CPUINFO_LIBRARY_TYPE "static" CACHE STRING "")
+      add_subdirectory(
+        "${QNNPACK_SOURCE_DIR}"
+        "${CONFU_DEPENDENCIES_BINARY_DIR}/QNNPACK")
+      # We build static versions of QNNPACK and pthreadpool but link
+      # them into a shared library for Caffe2, so they need PIC.
+      set_property(TARGET qnnpack PROPERTY POSITION_INDEPENDENT_CODE ON)
+      set_property(TARGET pthreadpool PROPERTY POSITION_INDEPENDENT_CODE ON)
+      set_property(TARGET cpuinfo PROPERTY POSITION_INDEPENDENT_CODE ON)
+    endif()
+
+    list(APPEND Caffe2_DEPENDENCY_LIBS qnnpack)
+  endif()
+endif()
+
+# ---[ Caffe2 Int8 operators (enabled by USE_QNNPACK) depend on gemmlowp and neon2sse headers
+if(USE_QNNPACK)
+  set(CAFFE2_THIRD_PARTY_ROOT "${PROJECT_SOURCE_DIR}/third_party")
+  include_directories(SYSTEM "${CAFFE2_THIRD_PARTY_ROOT}/gemmlowp")
+  include_directories(SYSTEM "${CAFFE2_THIRD_PARTY_ROOT}/neon2sse")
+endif()
+
 # ---[ NNPACK
 if(USE_NNPACK)
  include(${CMAKE_CURRENT_LIST_DIR}/External/nnpack.cmake)
--- a/cmake/Summary.cmake
+++ b/cmake/Summary.cmake
@ -115,6 +115,7 @@ function (caffe2_print_configuration_summary)
  endif()
  message(STATUS "  USE_OPENMP            : ${USE_OPENMP}")
  message(STATUS "  USE_PROF              : ${USE_PROF}")
+  message(STATUS "  USE_QNNPACK           : ${USE_QNNPACK}")
  message(STATUS "  USE_REDIS             : ${USE_REDIS}")
  message(STATUS "  USE_ROCKSDB           : ${USE_ROCKSDB}")
  message(STATUS "  USE_ZMQ               : ${USE_ZMQ}")
--- a/setup.py
+++ b/setup.py
@ -39,6 +39,9 @@
 #   NO_NNPACK
 #     disables NNPACK build
 #
+#   NO_QNNPACK
+#     disables QNNPACK build (quantized 8-bit operators)
+#
 #   NO_DISTRIBUTED
 #     disables distributed (c10d, gloo, mpi, etc.) build
 #
@ -156,7 +159,7 @@ def hotpatch_var(var, prefix='USE_'):
 # Before we run the setup_helpers, let's look for NO_* and WITH_*
 # variables and hotpatch environment with the USE_* equivalent
 use_env_vars = ['CUDA', 'CUDNN', 'MIOPEN', 'MKLDNN', 'NNPACK', 'DISTRIBUTED',
-                'OPENCV', 'FFMPEG', 'SYSTEM_NCCL', 'GLOO_IBVERBS']
+                'OPENCV', 'QNNPACK', 'FFMPEG', 'SYSTEM_NCCL', 'GLOO_IBVERBS']
 list(map(hotpatch_var, use_env_vars))

 # Also hotpatch a few with BUILD_* equivalent
@ -177,6 +180,7 @@ from tools.setup_helpers.nccl import USE_NCCL, USE_SYSTEM_NCCL, NCCL_LIB_DIR, \
 from tools.setup_helpers.mkldnn import (USE_MKLDNN, MKLDNN_LIBRARY,
                                        MKLDNN_LIB_DIR, MKLDNN_INCLUDE_DIR)
 from tools.setup_helpers.nnpack import USE_NNPACK
+from tools.setup_helpers.qnnpack import USE_QNNPACK
 from tools.setup_helpers.nvtoolext import NVTOOLEXT_HOME
 from tools.setup_helpers.generate_code import generate_code
 from tools.setup_helpers.ninja_builder import NinjaBuilder, ninja_build_ext
@ -396,6 +400,8 @@ def build_libs(libs):
        my_env["MKLDNN_LIBRARY"] = MKLDNN_LIBRARY
        my_env["MKLDNN_INCLUDE_DIR"] = MKLDNN_INCLUDE_DIR
        build_libs_cmd += ['--use-mkldnn']
+    if USE_QNNPACK:
+        build_libs_cmd += ['--use-qnnpack']
    if USE_GLOO_IBVERBS:
        build_libs_cmd += ['--use-gloo-ibverbs']
    if not RERUN_CMAKE:
--- a/third_party/QNNPACK
+++ b/third_party/QNNPACK
--- a/tools/build_pytorch_libs.bat
+++ b/tools/build_pytorch_libs.bat
@ -40,6 +40,13 @@ IF "%~1"=="--use-nnpack" (
  set /a USE_NNPACK=0
 )

+IF "%~1"=="--use-qnnpack" (
+  set /a USE_QNNPACK=1
+  shift
+) ELSE (
+  set /a USE_QNNPACK=0
+)
+
 IF "%~1"=="--use-mkldnn" (
  set /a NO_MKLDNN=0
  shift
@ -190,6 +197,7 @@ goto:eof
                  -DUSE_LEVELDB=%USE_LEVELDB% ^
                  -DUSE_LMDB=%USE_LMDB% ^
                  -DUSE_OPENCV=%USE_OPENCV% ^
+                  -DUSE_QNNPACK=%USE_QNNPACK% ^
                  -DUSE_FFMPEG=%USE_FFMPEG% ^
                  -DUSE_GLOG=OFF ^
                  -DUSE_GFLAGS=OFF ^
--- a/tools/build_pytorch_libs.sh
+++ b/tools/build_pytorch_libs.sh
@ -46,6 +46,7 @@ USE_CUDA=0
 USE_ROCM=0
 USE_NNPACK=0
 USE_MKLDNN=0
+USE_QNNPACK=0
 USE_GLOO_IBVERBS=0
 CAFFE2_STATIC_LINK_CUDA=0
 RERUN_CMAKE=1
@ -66,6 +67,9 @@ while [[ $# -gt 0 ]]; do
      --use-mkldnn)
          USE_MKLDNN=1
          ;;
+      --use-qnnpack)
+          USE_QNNPACK=1
+          ;;
      --use-gloo-ibverbs)
          USE_GLOO_IBVERBS=1
          ;;
@ -315,6 +319,7 @@ function build_caffe2() {
 		       -DUSE_LEVELDB=$USE_LEVELDB \
 		       -DUSE_LMDB=$USE_LMDB \
 		       -DUSE_OPENCV=$USE_OPENCV \
+		       -DUSE_QNNPACK=$USE_QNNPACK \
 		       -DUSE_FFMPEG=$USE_FFMPEG \
 		       -DUSE_GLOG=OFF \
 		       -DUSE_GFLAGS=OFF \
--- a/tools/setup_helpers/qnnpack.py
+++ b/tools/setup_helpers/qnnpack.py
@ -0,0 +1,6 @@
+from .env import check_env_flag
+
+if check_env_flag('NO_QNNPACK'):
+    USE_QNNPACK = False
+else:
+    USE_QNNPACK = True