[T153220354] Fix header inclusions in c10 (#1541) (#101846)

Summary:
This is a re-attempt to land the iwyu header changes, by taking the diff from [PR 100304](https://github.com/pytorch/pytorch/pull/100304), and adding the bare minimal changes to make the diff build corectly in the internal builds.

X-link: https://github.com/facebookresearch/pytorch3d/pull/1541

X-link: https://github.com/fairinternal/pytorch3d/pull/44

- Re-work D45769819 to fix header inclusions in c10

Test Plan:
```
buck2 build --no-remote-cache mode/dev-nosan //caffe2/c10/...

buck2 build --no-remote-cache mode/dev-nosan //deeplearning/fbgemm/fbgemm_gpu/...

buck2 build mode/dev-nosan //vision/fair/pytorch3d/pytorch3d:_C
```

Reviewed By: malfet

Differential Revision: D45920611

Pull Request resolved: https://github.com/pytorch/pytorch/pull/101846
Approved by: https://github.com/malfet, https://github.com/Skylion007
This commit is contained in:
Benson Ma
2023-05-20 19:35:14 +00:00
committed by PyTorch MergeBot
parent dde6d56101
commit 66a2600b6a
73 changed files with 88 additions and 105 deletions

View File

@ -1,7 +1,7 @@
#include <ATen/ATen.h>
#include <ATen/CachedTensorUtils.h>
#include <c10/util/flat_hash_map.h>
namespace at {
namespace caching {

View File

@ -4,6 +4,7 @@
#include <exception>
#include <mutex>
#include <ATen/CachedTensorUtils.h>
#include <c10/util/flat_hash_map.h>
namespace at {
namespace autocast {

View File

@ -12,6 +12,7 @@
#include <c10/core/UndefinedTensorImpl.h>
#include <c10/core/WrapDimMinimal.h>
#include <c10/util/Exception.h>
#include <c10/util/ExclusivelyOwned.h>
#include <c10/util/ExclusivelyOwnedTensorTraits.h>
#include <c10/util/MaybeOwned.h>
#include <c10/util/Optional.h>

View File

@ -14,6 +14,8 @@
#include <c10/util/MaybeOwned.h>
#include <c10/util/intrusive_ptr.h>
#include <typeindex>
#include <unordered_map>
#include <unordered_set>
#include <utility>
namespace torch {

View File

@ -22,6 +22,7 @@
#include <c10/core/UndefinedTensorImpl.h>
#include <c10/core/impl/DeviceGuardImplInterface.h>
#include <c10/util/FunctionRef.h>
#include <c10/util/Logging.h>
#include <c10/util/hash.h>
#include <c10/util/intrusive_ptr.h>
#include <c10/util/irange.h>

View File

@ -1,5 +1,6 @@
#include <ATen/core/Tensor.h>
#include <ATen/core/jit_type.h>
#include <c10/core/GradMode.h>
#include <utility>

View File

@ -7,6 +7,7 @@
#include <ATen/core/grad_mode.h>
#include <ATen/core/jit_type.h>
#include <c10/macros/Macros.h>
#include <c10/util/flat_hash_map.h>
#include <c10/util/irange.h>
#include <array>
#include <iostream>

View File

@ -14,6 +14,7 @@
#include <ATen/Context.h>
#include <c10/cuda/CUDAStream.h>
#include <c10/cuda/CUDAFunctions.h>
#include <c10/util/Logging.h>
#include <ATen/cuda/Exceptions.h>
namespace at {

View File

@ -3,7 +3,11 @@
#include <c10/cuda/CUDACachingAllocator.h>
#include <map>
#include <memory>
#include <regex>
#include <string>
#include <tuple>
namespace at { namespace cuda {

View File

@ -1,5 +1,6 @@
#define TORCH_ASSERT_ONLY_METHOD_OPERATORS
#include <ATen/core/Tensor.h>
#include <ATen/core/grad_mode.h>
#include <ATen/Dispatch.h>
#include <ATen/Parallel.h>
#include <ATen/TensorMeta.h>

View File

@ -9,6 +9,7 @@
#include <ATen/native/cpu/DepthwiseConvKernel.h>
#include <ATen/native/utils/ParamUtils.h>
#include <ATen/native/xnnpack/Engine.h>
#include <c10/core/GradMode.h>
#include <c10/util/accumulate.h>
#include <c10/util/irange.h>
#include <c10/macros/Macros.h>

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/DeviceType.h>
#include <c10/macros/Export.h>
#include <c10/macros/Macros.h>
#include <atomic>
#include <utility>

View File

@ -1,5 +1,6 @@
#define TORCH_ASSERT_ONLY_METHOD_OPERATORS
#include <ATen/core/Tensor.h>
#include <ATen/core/grad_mode.h>
#include <ATen/ExpandUtils.h>
#include <ATen/NamedTensorUtils.h>
#include <ATen/TensorOperators.h>

View File

@ -9,6 +9,8 @@
#include <ATen/native/quantized/PackedParams.h>
#include <ATen/native/quantized/cpu/fbgemm_utils.h>
#include <ATen/native/quantized/cpu/QnnpackUtils.h>
#include <c10/core/GradMode.h>
#include <c10/macros/Macros.h>
#include <c10/util/irange.h>
#include <torch/custom_class.h>
#include <torch/library.h>

View File

@ -2,6 +2,7 @@
#include <ATen/native/DispatchStub.h>
#include <ATen/Generator.h>
#include <c10/core/Scalar.h>
#include <stdexcept>
namespace at {

View File

@ -6,6 +6,8 @@
#include <ATen/TensorUtils.h>
#include <ATen/Dispatch.h>
#include <c10/core/GradMode.h>
#include <c10/macros/Macros.h>
#include <c10/util/Exception.h>
#ifndef AT_PER_OPERATOR_HEADERS

View File

@ -10,6 +10,7 @@
#include <ATen/TensorOperators.h>
#include <ATen/TensorUtils.h>
#include <ATen/core/Tensor.h>
#include <ATen/core/grad_mode.h>
#include <ATen/native/layer_norm.h>
#include <ATen/native/nested/NestedTensorUtils.h>

View File

@ -1,7 +1,6 @@
#include <ATen/ATen.h>
#include <ATen/NativeFunctions.h>
#include <ATen/native/TensorIterator.h>
#include <ATen/native/cpu/Loops.h>
#include <ATen/native/quantized/cpu/QuantUtils.h>
#include <ATen/quantized/QTensorImpl.h>
#include <ATen/quantized/Quantizer.h>

View File

@ -1,6 +1,5 @@
#include <ATen/ATen.h>
#include <ATen/CPUApplyUtils.h>
#include <ATen/Dispatch.h>
#include <ATen/ExpandUtils.h>
#include <ATen/NativeFunctions.h>
#include <ATen/native/ReduceOpsUtils.h>

View File

@ -4,6 +4,7 @@
#include <ATen/Parallel.h>
#include <ATen/SparseCsrTensorUtils.h>
#include <ATen/core/Tensor.h>
#include <ATen/core/grad_mode.h>
#include <ATen/mkl/Sparse.h>
#include <ATen/native/BinaryOps.h>
#include <ATen/native/CPUBlas.h>

View File

@ -3,6 +3,7 @@
#include <ATen/TensorSubclassLikeUtils.h>
#include <ATen/TensorUtils.h>
#include <ATen/core/Tensor.h>
#include <ATen/core/grad_mode.h>
#include <ATen/cuda/CUDAContext.h>
#include <ATen/detail/CUDAHooksInterface.h>
#include <ATen/native/DispatchStub.h>

View File

@ -21,6 +21,7 @@
#include <c10/core/UndefinedTensorImpl.h>
#include <c10/core/WrapDimMinimal.h>
#include <c10/util/Exception.h>
#include <c10/util/ExclusivelyOwned.h>
#include <c10/util/Deprecated.h>
#include <c10/util/MaybeOwned.h>
#include <c10/util/Optional.h>

View File

@ -54,6 +54,26 @@ target_compile_options(c10 PRIVATE "-DC10_BUILD_MAIN_LIB")
if(${COMPILER_SUPPORTS_HIDDEN_VISIBILITY})
target_compile_options(c10 PRIVATE "-fvisibility=hidden")
endif()
option(C10_USE_IWYU "Use include-what-you-use to clean up header inclusion" OFF)
if(C10_USE_IWYU)
find_program(iwyu NAMES include-what-you-use)
if(iwyu)
set(iwyu_cmd
"include-what-you-use"
"-Xiwyu"
"--transitive_includes_only"
"-Xiwyu"
"--no_fwd_decls"
"-Xiwyu"
"--prefix_header_includes=keep"
"-Xiwyu"
"--mapping_file=${CMAKE_CURRENT_LIST_DIR}/../tools/iwyu/all.imp"
)
set_property(TARGET c10 PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${iwyu_cmd})
endif()
endif()
if(WERROR)
target_compile_options_if_supported(c10 PRIVATE "-Werror=sign-compare")
target_compile_options_if_supported(c10 PRIVATE "-Werror=shadow")

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/ScalarType.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
namespace caffe2 {
class TypeMeta;

View File

@ -1,12 +1,10 @@
#include <c10/core/Device.h>
#include <c10/macros/Macros.h>
#include <c10/util/Exception.h>
#include <algorithm>
#include <array>
#include <cctype>
#include <exception>
#include <ostream>
#include <string>
#include <vector>

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/DeviceType.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <c10/util/Exception.h>
#include <cstddef>

View File

@ -5,7 +5,7 @@
// ATen/core (which would require a lot more build system hacking.)
// If you modify me, keep me synchronized with that file.
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <functional>
#include <ostream>

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/DeviceType.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <ostream>
#include <string>

View File

@ -1,6 +1,5 @@
#include <c10/core/DispatchKeySet.h>
#include <c10/util/irange.h>
#include <iostream>
namespace c10 {

View File

@ -3,6 +3,7 @@
#include <c10/util/Exception.h>
#include <c10/util/Metaprogramming.h>
#include <c10/util/llvmMathExtras.h>
#include <array>
#include <ostream>
namespace c10 {

View File

@ -8,6 +8,7 @@
#ifndef _WIN32
#include <fcntl.h>
#include <unistd.h>
#else
#include <chrono>
#endif

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/AutogradState.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
namespace c10 {

View File

@ -1,9 +1,8 @@
#pragma once
#include <c10/core/AutogradState.h>
#include <c10/core/GradMode.h>
#include <c10/core/impl/LocalDispatchKeySet.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
namespace c10 {

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/impl/PyInterpreter.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <c10/util/python_stub.h>
namespace c10 {

View File

@ -1,9 +1,7 @@
#pragma once
#include <assert.h>
#include <stdint.h>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <utility>

View File

@ -1,6 +1,7 @@
#pragma once
#include <c10/util/BFloat16.h>
#include <c10/util/Deprecated.h>
#include <c10/util/Exception.h>
#include <c10/util/Half.h>
#include <c10/util/bits.h>

View File

@ -1,7 +1,6 @@
#pragma once
#include <c10/core/Allocator.h>
#include <c10/core/ScalarType.h>
#include <c10/core/SymInt.h>
#include <c10/core/impl/PyObjectSlot.h>

View File

@ -7,7 +7,6 @@
#include <c10/util/intrusive_ptr.h>
#include <limits>
#include <memory>
namespace c10 {

View File

@ -3,9 +3,7 @@
#include <c10/core/SymInt.h>
#include <c10/core/SymNodeImpl.h>
#include <c10/util/intrusive_ptr.h>
#include <array>
#include <functional>
#include <utility>
namespace c10 {

View File

@ -7,6 +7,7 @@
#include <c10/util/Optional.h>
#include <numeric>
#include <type_traits>
namespace c10 {

View File

@ -1,10 +1,9 @@
#pragma once
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <c10/util/ArrayRef.h>
#include <c10/util/Exception.h>
#include <c10/util/intrusive_ptr.h>
#include <memory>
namespace c10 {

View File

@ -1,13 +1,14 @@
#include <c10/core/TensorImpl.h>
#include <c10/core/Backend.h>
#include <c10/core/CopyBytes.h>
#include <c10/core/InferenceMode.h>
#include <c10/core/SymIntArrayRef.h>
#include <c10/core/WrapDimMinimal.h>
#include <c10/core/impl/LocalDispatchKeySet.h>
#include <c10/core/impl/PyInterpreter.h>
#include <c10/core/impl/TorchDispatchModeTLS.h>
#include <c10/util/Logging.h>
#include <c10/util/Optional.h>
#include <c10/util/accumulate.h>
#include <c10/util/irange.h>
#include <utility>

View File

@ -1,34 +1,30 @@
#pragma once
#include <c10/core/Backend.h>
#include <c10/core/CopyBytes.h>
#include <c10/core/DispatchKeySet.h>
#include <c10/core/InferenceMode.h>
#include <c10/core/MemoryFormat.h>
#include <c10/core/ScalarTypeToTypeMeta.h>
#include <c10/core/Storage.h>
#include <c10/core/SymBool.h>
#include <c10/core/SymIntArrayRef.h>
#include <c10/core/TensorOptions.h>
#include <c10/core/WrapDimMinimal.h>
#include <c10/core/impl/LocalDispatchKeySet.h>
#include <c10/core/impl/PyObjectSlot.h>
#include <c10/core/impl/SizesAndStrides.h>
#include <c10/macros/Macros.h>
#include <c10/util/DimVector.h>
#include <c10/util/Exception.h>
#include <c10/util/Flags.h>
#include <c10/util/Logging.h>
#include <c10/util/Optional.h>
#include <c10/util/accumulate.h>
#include <c10/util/intrusive_ptr.h>
#include <c10/util/irange.h>
#include <c10/util/python_stub.h>
#include <c10/util/safe_numerics.h>
#include <c10/util/typeid.h>
#include <algorithm>
#include <atomic>
#include <limits>
#include <memory>
#include <numeric>
#include <type_traits>
#include <utility>
@ -57,11 +53,6 @@ class Tensor;
class TensorBase;
} // namespace at
namespace c10 {
class Scalar;
struct Storage;
} // namespace c10
namespace c10 {
/**
@ -151,8 +142,6 @@ struct C10_API PlacementDeleteContext {
}
};
struct TensorImpl;
struct C10_API AutogradMetaInterface {
virtual void set_requires_grad(
bool requires_grad,
@ -418,20 +407,6 @@ struct C10_API VariableVersion {
// C10_TensorImpl_Size_Check_Dummy_Class
struct C10_API TensorImpl;
// Forward declaration needed because TensorImpl needs to be friends with
// C10_TensorImpl_Size_Check_Dummy_Class in order to check the size
// of its private fields.
template <
size_t cplusplus,
size_t clang_ver_major,
size_t gcc_ver,
size_t gcc_ver_minor,
size_t nvcc,
size_t cuda_version,
size_t cuda_version_major,
size_t ptr_size>
class C10_TensorImpl_Size_Check_Dummy_Class;
/**
* NOTE: Some TensorImpl methods are small and not overridden in the
* PyTorch codebase itself, but may theoretically need to be

View File

@ -2,7 +2,6 @@
#include <c10/core/Device.h>
#include <c10/core/Layout.h>
#include <c10/core/ScalarType.h>
#include <c10/util/Optional.h>
#include <iostream>

View File

@ -3,17 +3,14 @@
#include <c10/core/Backend.h>
#include <c10/core/DefaultDtype.h>
#include <c10/core/Device.h>
#include <c10/core/DispatchKeySet.h>
#include <c10/core/Layout.h>
#include <c10/core/MemoryFormat.h>
#include <c10/core/ScalarType.h>
#include <c10/core/ScalarTypeToTypeMeta.h>
#include <c10/macros/Macros.h>
#include <c10/util/C++17.h>
#include <c10/util/Optional.h>
#include <cstddef>
#include <iosfwd>
#include <utility>

View File

@ -1,7 +1,6 @@
#pragma once
#include <c10/core/SymInt.h>
#include <c10/util/Exception.h>
namespace c10 {

View File

@ -1,5 +1,3 @@
#include <mutex>
#include <c10/core/impl/GPUTrace.h>
#include <c10/util/CallOnce.h>

View File

@ -1,6 +1,6 @@
#pragma once
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <atomic>
namespace c10 {

View File

@ -4,7 +4,7 @@
#include <c10/core/Layout.h>
#include <c10/core/MemoryFormat.h>
#include <c10/core/SymIntArrayRef.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <c10/util/ArrayRef.h>
#include <c10/util/intrusive_ptr.h>
#include <c10/util/python_stub.h>
@ -17,7 +17,6 @@ namespace c10 {
struct IValue;
class OperatorHandle;
struct TensorImpl;
struct SafePyObject;
} // namespace c10
namespace torch {

View File

@ -1,5 +1,4 @@
#include <c10/core/DispatchKeySet.h>
#include <c10/core/SafePyObject.h>
#include <c10/core/DispatchKey.h>
#include <c10/core/impl/LocalDispatchKeySet.h>
#include <c10/core/impl/PythonDispatcherTLS.h>

View File

@ -1,8 +1,7 @@
#pragma once
#include <c10/core/SafePyObject.h>
#include <c10/macros/Macros.h>
#include <c10/util/Optional.h>
#include <c10/core/impl/PyInterpreter.h>
#include <c10/macros/Export.h>
namespace c10 {
namespace impl {

View File

@ -1,4 +1,4 @@
#include <c10/core/DispatchKeySet.h>
#include <c10/core/DispatchKey.h>
#include <c10/core/SafePyObject.h>
#include <c10/core/impl/LocalDispatchKeySet.h>
#include <c10/core/impl/TorchDispatchModeTLS.h>

View File

@ -1,7 +1,7 @@
#pragma once
#include <c10/core/SafePyObject.h>
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
namespace c10 {
namespace impl {

View File

@ -1,6 +1,6 @@
#pragma once
#include <c10/macros/Macros.h>
#include <c10/macros/Export.h>
#include <cstddef>

View File

@ -1,4 +1,5 @@
#include <c10/core/thread_pool.h>
#include <c10/util/Logging.h>
namespace c10 {

View File

@ -1,5 +1,3 @@
#include <climits>
#include <c10/core/impl/alloc_cpu.h>
#include <c10/mobile/CPUProfilingAllocator.h>
#include <c10/util/irange.h>

View File

@ -1,13 +1,8 @@
#pragma once
#include <algorithm>
#include <deque>
#include <memory>
#include <mutex>
#include <c10/util/Exception.h>
#include <c10/util/SmallVector.h>
#include <c10/util/flat_hash_map.h>
#include <memory>
#include <vector>
namespace c10 {

View File

@ -1,4 +1,3 @@
#include <c10/util/Backtrace.h>
#include <c10/util/Exception.h>
#include <c10/util/Logging.h>
#include <c10/util/Type.h>

View File

@ -2,14 +2,11 @@
#define C10_UTIL_EXCEPTION_H_
#include <c10/macros/Macros.h>
#include <c10/util/Deprecated.h>
#include <c10/util/StringUtil.h>
#include <c10/util/variant.h>
#include <cstddef>
#include <exception>
#include <ostream>
#include <sstream>
#include <string>
#include <vector>

View File

@ -3,9 +3,7 @@
#include <c10/util/Synchronized.h>
#include <array>
#include <atomic>
#include <functional>
#include <mutex>
#include <shared_mutex>
#include <thread>
namespace c10 {

View File

@ -2,7 +2,6 @@
#include <c10/util/Array.h>
#include <c10/util/TypeList.h>
#include <array>
#include <functional>
#include <type_traits>

View File

@ -3,7 +3,9 @@
#include <cstddef>
#include <string>
#ifdef __GXX_RTTI
#include <typeinfo>
#endif // __GXX_RTTI
#include <c10/macros/Macros.h>

View File

@ -1,7 +1,6 @@
#pragma once
#include <c10/util/C++17.h>
#include <functional>
namespace c10 {
namespace guts {

View File

@ -1,13 +1,10 @@
#pragma once
#include <c10/util/C++17.h>
#include <c10/util/Exception.h>
#include <c10/util/ExclusivelyOwned.h>
#include <c10/util/MaybeOwned.h>
#include <atomic>
#include <climits>
#include <memory>
#include <stdexcept>
namespace pybind11 {
template <typename, typename...>

View File

@ -1,3 +1,4 @@
#include <c10/util/Exception.h>
#include <c10/util/numa.h>
C10_DEFINE_bool(caffe2_cpu_numa_enabled, false, "Use NUMA whenever possible.");

View File

@ -1,6 +1,8 @@
#pragma once
#include <c10/util/Logging.h>
#include <c10/macros/Export.h>
#include <c10/util/Flags.h>
#include <stddef.h>
C10_DECLARE_bool(caffe2_cpu_numa_enabled);

View File

@ -1,30 +1,17 @@
#pragma once
#include <atomic>
#include <cassert>
#include <complex>
#include <cstdlib>
#include <memory>
#include <mutex>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#ifdef __GXX_RTTI
#include <typeinfo>
#endif
#include <exception>
#include <c10/macros/Macros.h>
#include <c10/util/Backtrace.h>
#include <c10/util/C++17.h>
#include <c10/util/Exception.h>
#include <c10/util/IdWrapper.h>
#include <c10/util/Type.h>
#include <c10/util/TypeIndex.h>
#include <c10/util/TypeTraits.h>
#include <c10/util/flat_hash_map.h>
#include <c10/core/ScalarType.h>
#include <c10/util/irange.h>

View File

@ -17,6 +17,7 @@
#include <ATen/dlpack.h>
#include <ATen/native/ConvUtils.h>
#include <c10/core/DispatchKeySet.h>
#include <c10/util/Backtrace.h>
#include <c10/util/Logging.h>
#include <c10/util/irange.h>
#include <libshm.h>

View File

@ -3,6 +3,7 @@
#include <ATen/Dispatch.h>
#include <ATen/ScalarOps.h>
#include <ATen/core/Tensor.h>
#include <ATen/core/grad_mode.h>
#include <c10/util/irange.h>

View File

@ -1,5 +1,7 @@
#include <torch/optim/schedulers/reduce_on_plateau_scheduler.h>
#include <iomanip>
namespace torch {
namespace optim {

View File

@ -1,6 +1,7 @@
#pragma once
#include <ATen/core/Tensor.h>
#include <unordered_set>
namespace torch {
namespace autograd {

View File

@ -5,6 +5,7 @@
#include <ATen/SparseCsrTensorUtils.h>
#include <ATen/TensorOperators.h>
#include <ATen/TensorSubclassLikeUtils.h>
#include <ATen/core/grad_mode.h>
#include <ATen/native/SparseTensorUtils.h>
#include <c10/core/DeviceGuard.h>

View File

@ -2,6 +2,7 @@
#include <ATen/core/dynamic_type.h>
#include <ATen/core/jit_type.h>
#include <unordered_set>
namespace c10 {