[BE][CMake] Use FindPython module (#124613)

As FindPythonInterp and FindPythonLibs has been deprecated since cmake-3.12

Replace `PYTHON_EXECUTABLE` with `Python_EXECUTABLE` everywhere (CMake variable names are case-sensitive)

This makes PyTorch buildable with python3 binary shipped with XCode on MacOS

TODO: Get rid of `FindNumpy` as its part of Python package
Pull Request resolved: https://github.com/pytorch/pytorch/pull/124613
Approved by: https://github.com/cyyever, https://github.com/Skylion007
This commit is contained in:
Nikita Shulga
2024-05-29 13:17:35 +00:00
committed by PyTorch MergeBot
parent 942d9abd66
commit 0910429d72
16 changed files with 53 additions and 93 deletions

View File

@ -330,7 +330,7 @@ else
SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
mkdir -p "$CUSTOM_OP_BUILD"
pushd "$CUSTOM_OP_BUILD"
cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)" \
cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPython_EXECUTABLE="$(which python)" \
-DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
make VERBOSE=1
popd
@ -343,7 +343,7 @@ else
SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
mkdir -p "$JIT_HOOK_BUILD"
pushd "$JIT_HOOK_BUILD"
cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)" \
cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPython_EXECUTABLE="$(which python)" \
-DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
make VERBOSE=1
popd
@ -355,7 +355,7 @@ else
python --version
mkdir -p "$CUSTOM_BACKEND_BUILD"
pushd "$CUSTOM_BACKEND_BUILD"
cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)" \
cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPython_EXECUTABLE="$(which python)" \
-DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
make VERBOSE=1
popd

View File

@ -388,7 +388,7 @@ add_custom_command(
OUTPUT
${TORCH_GENERATED_CODE}
COMMAND
"${PYTHON_EXECUTABLE}" tools/setup_helpers/generate_code.py
"${Python_EXECUTABLE}" tools/setup_helpers/generate_code.py
--native-functions-path "aten/src/ATen/native/native_functions.yaml"
--tags-path "aten/src/ATen/native/tags.yaml"
$<$<BOOL:${INTERN_DISABLE_AUTOGRAD}>:--disable-autograd>
@ -1144,7 +1144,7 @@ if(BUILD_LITE_INTERPRETER AND SELECTED_OP_LIST)
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/aten/src/ATen/selected_mobile_ops.h
COMMAND
"${PYTHON_EXECUTABLE}"
"${Python_EXECUTABLE}"
-m tools.code_analyzer.gen_oplist
--model_file_list_path "${SELECTED_OP_LIST}"
--output_dir "${CMAKE_BINARY_DIR}/aten/src/ATen"
@ -1159,7 +1159,7 @@ if(BUILD_LITE_INTERPRETER AND SELECTED_OP_LIST)
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/aten/src/ATen/selected_mobile_ops.h
COMMAND
"${PYTHON_EXECUTABLE}"
"${Python_EXECUTABLE}"
-m tools.lite_interpreter.gen_selected_mobile_ops_header
--yaml_file_path "${SELECTED_OP_LIST}"
--output_file_path "${CMAKE_BINARY_DIR}/aten/src/ATen"

View File

@ -114,7 +114,7 @@ if(INTERN_BUILD_ATEN_OPS)
file(GLOB_RECURSE all_unboxing_script "${CMAKE_CURRENT_LIST_DIR}/../tools/jit/*.py")
list(APPEND CUSTOM_BUILD_FLAGS --skip_dispatcher_op_registration)
set(GEN_UNBOXING_COMMAND
"${PYTHON_EXECUTABLE}" -m tools.jit.gen_unboxing
"${Python_EXECUTABLE}" -m tools.jit.gen_unboxing
--source-path ${CMAKE_CURRENT_LIST_DIR}/../aten/src/ATen
--install_dir ${CMAKE_BINARY_DIR}/aten/src/ATen
)
@ -158,7 +158,7 @@ if(INTERN_BUILD_ATEN_OPS)
endif()
set(GEN_COMMAND
"${PYTHON_EXECUTABLE}" -m torchgen.gen
"${Python_EXECUTABLE}" -m torchgen.gen
--source-path ${CMAKE_CURRENT_LIST_DIR}/../aten/src/ATen
--install_dir ${CMAKE_BINARY_DIR}/aten/src/ATen
${GEN_PER_OPERATOR_FLAG}
@ -356,7 +356,7 @@ function(append_filelist name outputvar)
${PROJECT_SOURCE_DIR}/build_variables.bzl
${PROJECT_BINARY_DIR}/caffe2/build_variables.bzl)
execute_process(
COMMAND "${PYTHON_EXECUTABLE}" -c
COMMAND "${Python_EXECUTABLE}" -c
"exec(open('${PROJECT_SOURCE_DIR}/build_variables.bzl').read());print(';'.join(['${_rootdir}' + x for x in ${name}]))"
WORKING_DIRECTORY "${_rootdir}"
RESULT_VARIABLE _retval

View File

@ -837,76 +837,43 @@ include_directories(SYSTEM ${EIGEN3_INCLUDE_DIR})
# ---[ Python + Numpy
if(BUILD_PYTHON)
# If not given a Python installation, then use the current active Python
if(NOT PYTHON_EXECUTABLE)
if(NOT Python_EXECUTABLE)
execute_process(
COMMAND "which" "python" RESULT_VARIABLE _exitcode OUTPUT_VARIABLE _py_exe)
COMMAND "which" "python3" RESULT_VARIABLE _exitcode OUTPUT_VARIABLE _py_exe)
if(${_exitcode} EQUAL 0)
if(NOT MSVC)
string(STRIP ${_py_exe} PYTHON_EXECUTABLE)
string(STRIP ${_py_exe} Python_EXECUTABLE)
endif()
message(STATUS "Setting Python to ${PYTHON_EXECUTABLE}")
message(STATUS "Setting Python to ${Python_EXECUTABLE}")
endif()
endif()
# Check that Python works
set(PYTHON_VERSION)
if(DEFINED PYTHON_EXECUTABLE)
if(DEFINED Python_EXECUTABLE)
execute_process(
COMMAND "${PYTHON_EXECUTABLE}" "--version"
COMMAND "${Python_EXECUTABLE}" "--version"
RESULT_VARIABLE _exitcode OUTPUT_VARIABLE PYTHON_VERSION)
if(NOT _exitcode EQUAL 0)
message(FATAL_ERROR "The Python executable ${PYTHON_EXECUTABLE} cannot be run. Make sure that it is an absolute path.")
message(FATAL_ERROR "The Python executable ${Python_EXECUTABLE} cannot be run. Make sure that it is an absolute path.")
endif()
if(PYTHON_VERSION)
string(REGEX MATCH "([0-9]+)\\.([0-9]+)" PYTHON_VERSION ${PYTHON_VERSION})
endif()
endif()
# Seed PYTHON_INCLUDE_DIR and PYTHON_LIBRARY to be consistent with the
# executable that we already found (if we didn't actually find an executable
# then these will just use "python", but at least they'll be consistent with
# each other).
if(NOT PYTHON_INCLUDE_DIR)
# TODO: Verify that sysconfig isn't inaccurate
pycmd_no_exit(_py_inc _exitcode "import sysconfig; print(sysconfig.get_path('include'))")
if("${_exitcode}" EQUAL 0 AND IS_DIRECTORY "${_py_inc}")
set(PYTHON_INCLUDE_DIR "${_py_inc}")
message(STATUS "Setting Python's include dir to ${_py_inc} from sysconfig")
else()
message(WARNING "Could not set Python's include dir to ${_py_inc} from sysconfig")
endif()
endif(NOT PYTHON_INCLUDE_DIR)
if(NOT PYTHON_LIBRARY)
pycmd_no_exit(_py_lib _exitcode "import sysconfig; print(sysconfig.get_path('stdlib'))")
if("${_exitcode}" EQUAL 0 AND EXISTS "${_py_lib}" AND EXISTS "${_py_lib}")
set(PYTHON_LIBRARY "${_py_lib}")
if(MSVC)
string(REPLACE "Lib" "libs" _py_static_lib ${_py_lib})
link_directories(${_py_static_lib})
endif()
message(STATUS "Setting Python's library to ${PYTHON_LIBRARY}")
endif()
endif(NOT PYTHON_LIBRARY)
# These should fill in the rest of the variables, like versions, but resepct
# the variables we set above
set(Python_ADDITIONAL_VERSIONS ${PYTHON_VERSION} 3.8)
find_package(PythonInterp 3.0)
find_package(PythonLibs 3.0)
find_package(Python COMPONENTS Interpreter Development)
if(NOT PYTHONLIBS_VERSION_STRING)
if(NOT Python_Development_FOUND)
message(FATAL_ERROR
"Python development libraries could not be found.")
endif()
if(${PYTHONLIBS_VERSION_STRING} VERSION_LESS 3)
if(${Python_VERSION} VERSION_LESS 3.8)
message(FATAL_ERROR
"Found Python libraries version ${PYTHONLIBS_VERSION_STRING}. Python 2 has reached end-of-life and is no longer supported by PyTorch.")
endif()
if(${PYTHONLIBS_VERSION_STRING} VERSION_LESS 3.8)
message(FATAL_ERROR
"Found Python libraries version ${PYTHONLIBS_VERSION_STRING}. Python < 3.8 is no longer supported by PyTorch.")
"Found Python libraries version ${Python_VERSION}. Python < 3.8 is no longer supported by PyTorch.")
endif()
# When building pytorch, we pass this in directly from setup.py, and
@ -920,11 +887,11 @@ if(BUILD_PYTHON)
endif()
endif()
if(PYTHONINTERP_FOUND AND PYTHONLIBS_FOUND)
if(Python_Interpreter_FOUND AND Python_Development_FOUND)
add_library(python::python INTERFACE IMPORTED)
target_include_directories(python::python SYSTEM INTERFACE ${PYTHON_INCLUDE_DIRS})
target_include_directories(python::python SYSTEM INTERFACE ${Python_INCLUDE_DIRS})
if(WIN32)
target_link_libraries(python::python INTERFACE ${PYTHON_LIBRARIES})
target_link_libraries(python::python INTERFACE ${Python_LIBRARIES})
endif()
caffe2_update_option(USE_NUMPY OFF)
@ -960,10 +927,7 @@ endif()
message(STATUS "pybind11 include dirs: " "${pybind11_INCLUDE_DIRS}")
add_library(pybind::pybind11 INTERFACE IMPORTED)
target_include_directories(pybind::pybind11 SYSTEM INTERFACE ${pybind11_INCLUDE_DIRS})
target_link_libraries(pybind::pybind11 INTERFACE python::python)
if(APPLE)
target_link_options(pybind::pybind11 INTERFACE -undefined dynamic_lookup)
endif()
target_link_libraries(pybind::pybind11 INTERFACE Python::Module)
# ---[ OpenTelemetry API headers
find_package(OpenTelemetryApi)

View File

@ -15,8 +15,8 @@
unset(NUMPY_VERSION)
unset(NUMPY_INCLUDE_DIR)
if(PYTHONINTERP_FOUND)
execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
if(Python_Interpreter_FOUND)
execute_process(COMMAND "${Python_EXECUTABLE}" "-c"
"import numpy as n; print(n.__version__); print(n.get_include());"
RESULT_VARIABLE __result
OUTPUT_VARIABLE __output

View File

@ -36,12 +36,11 @@ function(caffe2_print_configuration_summary)
message(STATUS " BUILD_DOCS : ${BUILD_DOCS}")
message(STATUS " BUILD_PYTHON : ${BUILD_PYTHON}")
if(${BUILD_PYTHON})
message(STATUS " Python version : ${PYTHON_VERSION_STRING}")
message(STATUS " Python executable : ${PYTHON_EXECUTABLE}")
message(STATUS " Pythonlibs version : ${PYTHONLIBS_VERSION_STRING}")
message(STATUS " Python library : ${PYTHON_LIBRARIES}")
message(STATUS " Python includes : ${PYTHON_INCLUDE_DIRS}")
message(STATUS " Python site-packages: ${PYTHON_SITE_PACKAGES}")
message(STATUS " Python version : ${Python_VERSION}")
message(STATUS " Python executable : ${Python_EXECUTABLE}")
message(STATUS " Python library : ${Python_LIBRARIES}")
message(STATUS " Python includes : ${Python_INCLUDE_DIRS}")
message(STATUS " Python site-package : ${Python_SITELIB}")
endif()
message(STATUS " BUILD_SHARED_LIBS : ${BUILD_SHARED_LIBS}")
message(STATUS " CAFFE2_USE_MSVC_STATIC_RUNTIME : ${CAFFE2_USE_MSVC_STATIC_RUNTIME}")

View File

@ -42,7 +42,7 @@ list(APPEND NEW_PYTHONPATH "${CMAKE_CURRENT_LIST_DIR}/..")
set(ENV{PYTHONPATH} ${NEW_PYTHONPATH})
execute_process(
COMMAND
"${PYTHON_EXECUTABLE}"
"${Python_EXECUTABLE}"
${CMAKE_CURRENT_LIST_DIR}/../tools/gen_vulkan_spv.py
--glsl-path ${CMAKE_CURRENT_LIST_DIR}/../aten/src/ATen/native/vulkan/glsl
--output-path ${VULKAN_GEN_OUTPUT_PATH}

View File

@ -139,10 +139,10 @@ endif()
set(CUDA_CUDA_LIB "${CUDA_cuda_driver_LIBRARY}" CACHE FILEPATH "")
set(CUDA_NVRTC_LIB "${CUDA_nvrtc_LIBRARY}" CACHE FILEPATH "")
if(CUDA_NVRTC_LIB AND NOT CUDA_NVRTC_SHORTHASH)
if("${PYTHON_EXECUTABLE}" STREQUAL "")
set(_python_exe "python")
if("${Python_EXECUTABLE}" STREQUAL "")
set(_python_exe "python3")
else()
set(_python_exe "${PYTHON_EXECUTABLE}")
set(_python_exe "${Python_EXECUTABLE}")
endif()
execute_process(
COMMAND "${_python_exe}" -c

View File

@ -101,7 +101,7 @@ endfunction()
# setting to `python -c`, or using with pycmd. This allows multiline code to be
# nested nicely in the surrounding code structure.
#
# This function respsects PYTHON_EXECUTABLE if it defined, otherwise it uses
# This function respsects Python_EXECUTABLE if it defined, otherwise it uses
# `python` and hopes for the best. An error will be thrown if it is not found.
#
# Args:
@ -109,11 +109,11 @@ endfunction()
# text : text to remove indentation from
#
function(dedent outvar text)
# Use PYTHON_EXECUTABLE if it is defined, otherwise default to python
if("${PYTHON_EXECUTABLE}" STREQUAL "")
set(_python_exe "python")
# Use Python_EXECUTABLE if it is defined, otherwise default to python
if("${Python_EXECUTABLE}" STREQUAL "")
set(_python_exe "python3")
else()
set(_python_exe "${PYTHON_EXECUTABLE}")
set(_python_exe "${Python_EXECUTABLE}")
endif()
set(_fixup_cmd "import sys; from textwrap import dedent; print(dedent(sys.stdin.read()))")
file(WRITE "${CMAKE_BINARY_DIR}/indented.txt" "${text}")
@ -134,11 +134,11 @@ endfunction()
function(pycmd_no_exit outvar exitcode cmd)
# Use PYTHON_EXECUTABLE if it is defined, otherwise default to python
if("${PYTHON_EXECUTABLE}" STREQUAL "")
# Use Python_EXECUTABLE if it is defined, otherwise default to python
if("${Python_EXECUTABLE}" STREQUAL "")
set(_python_exe "python")
else()
set(_python_exe "${PYTHON_EXECUTABLE}")
set(_python_exe "${Python_EXECUTABLE}")
endif()
# run the actual command
execute_process(
@ -159,7 +159,7 @@ endfunction()
# Common indentation in the text of `cmd` is removed before the command is
# executed, so the caller does not need to worry about indentation issues.
#
# This function respsects PYTHON_EXECUTABLE if it defined, otherwise it uses
# This function respsects Python_EXECUTABLE if it defined, otherwise it uses
# `python` and hopes for the best. An error will be thrown if it is not found.
#
# Args:

View File

@ -61,7 +61,7 @@ CMAKE_ARGS=()
# Build PyTorch mobile
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DPython_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
# custom build with selected ops

View File

@ -26,7 +26,7 @@ CMAKE_ARGS=()
# Build PyTorch mobile
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DPython_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
# custom build with selected ops

View File

@ -16,7 +16,7 @@ echo "Caffe2 path: $CAFFE2_ROOT"
CMAKE_ARGS=()
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DPython_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")

View File

@ -17,8 +17,7 @@ import sys
import sysconfig
flags = [
f"-DPYTHON_EXECUTABLE:FILEPATH={sys.executable}",
f"-DPYTHON_INCLUDE_DIR={sysconfig.get_path('include')}",
f"-DPython_EXECUTABLE:FILEPATH={sys.executable}",
]
print(" ".join(flags), end="")

View File

@ -9,7 +9,7 @@ append_cxx_flag_if_supported("-Wno-unused-private-field" CMAKE_CXX_FLAGS)
# Generate unboxing kernels
set(GEN_COMMAND
"${PYTHON_EXECUTABLE}" -m torchgen.gen_executorch
"${Python_EXECUTABLE}" -m torchgen.gen_executorch
--source-path=${TEST_ROOT}
--install-dir=${OUTPUT_DIRECTORY}
--tags-path=${TORCH_ROOT}/aten/src/ATen/native/tags.yaml

View File

@ -307,9 +307,7 @@ class CMake:
CMake.defines(
args,
PYTHON_EXECUTABLE=sys.executable,
PYTHON_LIBRARY=cmake_python_library,
PYTHON_INCLUDE_DIR=sysconfig.get_path("include"),
Python_EXECUTABLE=sys.executable,
TORCH_BUILD_VERSION=version,
NUMPY_INCLUDE_DIR=NUMPY_INCLUDE_DIR,
**build_options,

View File

@ -216,7 +216,7 @@ add_custom_command(
"${TORCH_SRC_DIR}/_C/_VariableFunctions.pyi"
"${TORCH_SRC_DIR}/nn/functional.pyi"
COMMAND
"${PYTHON_EXECUTABLE}" -mtools.pyi.gen_pyi
"${Python_EXECUTABLE}" -mtools.pyi.gen_pyi
--native-functions-path "aten/src/ATen/native/native_functions.yaml"
--tags-path "aten/src/ATen/native/tags.yaml"
--deprecated-functions-path "tools/autograd/deprecated.yaml"
@ -238,7 +238,7 @@ add_custom_command(
OUTPUT
"${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi"
COMMAND
"${PYTHON_EXECUTABLE}" ${TORCH_SRC_DIR}/utils/data/datapipes/gen_pyi.py
"${Python_EXECUTABLE}" ${TORCH_SRC_DIR}/utils/data/datapipes/gen_pyi.py
DEPENDS
"${TORCH_SRC_DIR}/utils/data/datapipes/datapipe.pyi.in"
${datapipe_files}
@ -383,7 +383,7 @@ add_custom_command(
OUTPUT ${TORCH_SRC_DIR}/version.py
COMMAND "${CMAKE_COMMAND}" -E touch "${TOOLS_PATH}/generate_torch_version.py"
COMMAND
"${PYTHON_EXECUTABLE}" "${TOOLS_PATH}/generate_torch_version.py"
"${Python_EXECUTABLE}" "${TOOLS_PATH}/generate_torch_version.py"
--is-debug=${TORCH_VERSION_DEBUG}
--cuda-version=${CUDA_VERSION}
--hip-version=${HIP_VERSION}