diff --git a/.bazelignore b/.bazelignore index 61b5e9458df6..01fcdd0d8e05 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,3 +1,4 @@ # We do not use this library in our Bazel build. It contains an # infinitely recursing symlink that makes Bazel very unhappy. third_party/ittapi/ +third_party/opentelemetry-cpp diff --git a/.gitmodules b/.gitmodules index 7e1b09e591cd..c9b84a370167 100644 --- a/.gitmodules +++ b/.gitmodules @@ -149,3 +149,6 @@ [submodule "third_party/mimalloc"] path = third_party/mimalloc url = https://github.com/microsoft/mimalloc.git +[submodule "third_party/opentelemetry-cpp"] + path = third_party/opentelemetry-cpp + url = https://github.com/open-telemetry/opentelemetry-cpp.git diff --git a/WORKSPACE b/WORKSPACE index 59bc0998ddcb..8eabea571a57 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -71,6 +71,13 @@ http_archive( ], ) +http_archive( + name = "com_github_opentelemetry-cpp", + urls = [ + "https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.14.2.tar.gz", + ], +) + new_local_repository( name = "gloo", build_file = "//third_party:gloo.BUILD", @@ -155,6 +162,12 @@ new_local_repository( path = "third_party/kineto", ) +new_local_repository( + name = "opentelemetry-cpp", + build_file = "//third_party::opentelemetry-cpp.BUILD", + path = "third_party/opentelemetry-cpp", +) + new_patched_local_repository( name = "tbb", build_file = "//third_party:tbb.BUILD", diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 8fc7f51f6d63..d8aea3999eae 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -1156,6 +1156,16 @@ if(APPLE) target_link_options(pybind::pybind11 INTERFACE -undefined dynamic_lookup) endif() +# ---[ OpenTelemetry API headers +find_package(OpenTelemetryApi) +if(NOT OpenTelemetryApi_FOUND) + message(STATUS "Using third_party/opentelemetry-cpp.") + set(OpenTelemetryApi_INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/../third_party/opentelemetry-cpp/api/include) +endif() +message(STATUS "opentelemetry api include dirs: " "${OpenTelemetryApi_INCLUDE_DIRS}") +add_library(opentelemetry::api INTERFACE IMPORTED) +target_include_directories(opentelemetry::api SYSTEM INTERFACE ${OpenTelemetryApi_INCLUDE_DIRS}) + # ---[ MPI if(USE_MPI) find_package(MPI) diff --git a/cmake/Modules/FindOpenTelemetryApi.cmake b/cmake/Modules/FindOpenTelemetryApi.cmake new file mode 100644 index 000000000000..01334ed7047b --- /dev/null +++ b/cmake/Modules/FindOpenTelemetryApi.cmake @@ -0,0 +1,19 @@ +# Try to find the OpenTelemetry API headers +# OpenTelemetryApi_FOUND - system has OpenTelemetry API headers +# OpenTelemetryApi_INCLUDE_DIRS - the OpenTelemetry API headers dir + +find_path(OpenTelemetryApi_INCLUDE_DIR + NAMES include/opentelemetry/version.h + DOC "The directory where Open Telemetry API headers reside" +) + +set(OpenTelemetryApi_INCLUDE_DIRS ${OpenTelemetryApi_INCLUDE_DIR}) + +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args(OpenTelemetryApi + FOUND_VAR OpenTelemetryApi_FOUND + REQUIRED_VARS OpenTelemetryApi_INCLUDE_DIRS +) + +mark_as_advanced(OpenTelemetryApi_FOUND) diff --git a/third_party/LICENSES_BUNDLED.txt b/third_party/LICENSES_BUNDLED.txt index df5cfee1f4f9..23d990fe594c 100644 --- a/third_party/LICENSES_BUNDLED.txt +++ b/third_party/LICENSES_BUNDLED.txt @@ -1,4 +1,4 @@ -The Pytorch repository and source distributions bundle several libraries that are +The Pytorch repository and source distributions bundle several libraries that are compatibly licensed. We list these here. Name: DCGM @@ -41,11 +41,28 @@ License: Apache-2.0 Files: third_party/benchmark, third_party/onnx/third_party/benchmark, third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark, - third_party/protobuf/third_party/benchmark + third_party/protobuf/third_party/benchmark, + third_party/opentelemetry-cpp/third_party/benchmark For details, see: third_party/benchmark/LICENSE, third_party/onnx/third_party/benchmark/LICENSE, third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark/LICENSE, - third_party/protobuf/third_party/benchmark/LICENSE + third_party/protobuf/third_party/benchmark/LICENSE, + third_party/opentelemetry-cpp/third_party/benchmark/LICENSE + +Name: boost-vcpkg-helpers +License: MIT +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/boost-vcpkg-helpers + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/boost-vcpkg-helpers/LICENSE.txt + +Name: cJSON +License: MIT +Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/examples/rest/cJSON + For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/examples/rest/cJSON/LICENSE + +Name: catch2 +License: BSL-1.0 +Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/catch2 + For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/catch2/LICENSE.txt Name: clog License: BSD-2-Clause @@ -104,6 +121,16 @@ Files: third_party/kineto/libkineto/third_party/dynolog/third_party/json/test/th For details, see: third_party/kineto/libkineto/third_party/dynolog/third_party/json/test/thirdparty/doctest/LICENSE.txt, third_party/nlohmann/tests/thirdparty/doctest/LICENSE.txt +Name: duktape-1.5.2 +License: MIT +Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.5.2 + For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.5.2/LICENSE.txt + +Name: duktape-1.8.0 +License: MIT +Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.8.0 + For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.8.0/LICENSE.txt + Name: dynolog License: MIT Files: third_party/kineto/libkineto/third_party/dynolog @@ -114,22 +141,37 @@ License: BSD-3-Clause Files: third_party/eigen For details, see: third_party/eigen/COPYING.BSD +Name: etw +License: MIT +Files: third_party/opentelemetry-cpp/exporters/etw/include/opentelemetry/exporters/etw + For details, see: third_party/opentelemetry-cpp/exporters/etw/include/opentelemetry/exporters/etw/LICENSE + +Name: expected +License: MIT +Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/expected + For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/expected/LICENSE + Name: fbgemm License: BSD-3-Clause Files: third_party/fbgemm For details, see: third_party/fbgemm/LICENSE +Name: ffnvcodec +License: MIT with exception +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/ffnvcodec + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/ffnvcodec/LICENSE.txt + Name: flatbuffers License: Apache-2.0 Files: third_party/flatbuffers - For details, see: third_party/flatbuffers/LICENSE.txt + For details, see: third_party/flatbuffers/LICENSE Name: fmt License: MIT with exception Files: third_party/fmt, third_party/kineto/libkineto/third_party/dynolog/third_party/fmt, third_party/kineto/libkineto/third_party/fmt - For details, see: third_party/fmt/LICENSE.rst, + For details, see: third_party/fmt/LICENSE, third_party/kineto/libkineto/third_party/dynolog/third_party/fmt/LICENSE.rst, third_party/kineto/libkineto/third_party/fmt/LICENSE.rst @@ -149,12 +191,19 @@ Files: third_party/fbgemm/third_party/googletest/googlemock/scripts/generator, third_party/googletest/googlemock/scripts/generator, third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator, third_party/protobuf/third_party/googletest/googlemock/scripts/generator, - third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator + third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator, + third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest/googlemock/scripts/generator For details, see: third_party/fbgemm/third_party/googletest/googlemock/scripts/generator/LICENSE, third_party/googletest/googlemock/scripts/generator/LICENSE, third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator/LICENSE, third_party/protobuf/third_party/googletest/googlemock/scripts/generator/LICENSE, - third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator/LICENSE + third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator/LICENSE, + third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest/googlemock/scripts/generator/LICENSE + +Name: gettimeofday +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/gettimeofday + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/gettimeofday/LICENSE Name: gloo License: BSD-3-Clause @@ -183,7 +232,9 @@ Files: third_party/fbgemm/third_party/googletest, third_party/protobuf/third_party/googletest, third_party/protobuf/third_party/googletest/googletest, third_party/tensorpipe/third_party/googletest, - third_party/tensorpipe/third_party/googletest/googletest + third_party/tensorpipe/third_party/googletest/googletest, + third_party/opentelemetry-cpp/third_party/googletest, + third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest For details, see: third_party/fbgemm/third_party/googletest/LICENSE, third_party/fbgemm/third_party/googletest/googletest/LICENSE, third_party/googletest/LICENSE, @@ -193,20 +244,25 @@ Files: third_party/fbgemm/third_party/googletest, third_party/protobuf/third_party/googletest/LICENSE, third_party/protobuf/third_party/googletest/googletest/LICENSE, third_party/tensorpipe/third_party/googletest/LICENSE, - third_party/tensorpipe/third_party/googletest/googletest/LICENSE + third_party/tensorpipe/third_party/googletest/googletest/LICENSE, + third_party/opentelemetry-cpp/third_party/googletest/LICENSE, + third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest/LICENSE Name: gtest License: BSD-3-Clause -Files: third_party/ideep/mkl-dnn/tests/gtest, - third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest - For details, see: third_party/ideep/mkl-dnn/tests/gtest/LICENSE, - third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest/LICENSE +Files: third_party/ideep/mkl-dnn/tests/gtests/gtest + For details, see: third_party/ideep/mkl-dnn/tests/gtests/gtest/LICENSE Name: hipify_torch License: MIT Files: third_party/fbgemm/third_party/hipify_torch For details, see: third_party/fbgemm/third_party/hipify_torch/LICENSE.txt +Name: hungarian +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/hungarian + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/hungarian/LICENSE.txt + Name: ideep License: MIT Files: third_party/ideep @@ -217,10 +273,10 @@ License: BSD-3-Clause Files: third_party/ios-cmake For details, see: third_party/ios-cmake/LICENSE -Name: json +Name: irrlicht License: MIT -Files: third_party/cudnn_frontend/include/contrib/nlohmann/json - For details, see: third_party/cudnn_frontend/include/contrib/nlohmann/json/LICENSE.txt +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/irrlicht + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/irrlicht/LICENSE.txt Name: kineto License: BSD-3-Clause @@ -232,11 +288,21 @@ License: Apache-2.0 Files: third_party/tensorpipe/third_party/libnop For details, see: third_party/tensorpipe/third_party/libnop/LICENSE +Name: libstemmer +License: BSD-3-Clause +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/libstemmer + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/libstemmer/LICENSE + Name: libuv License: MIT Files: third_party/tensorpipe/third_party/libuv For details, see: third_party/tensorpipe/third_party/libuv/LICENSE +Name: mimalloc +License: MIT +Files: third_party/mimalloc + For details, see: third_party/mimalloc/LICENSE + Name: miniz-2.1.0 License: MIT Files: third_party/miniz-2.1.0 @@ -247,6 +313,11 @@ License: Apache-2.0 Files: third_party/ideep/mkl-dnn For details, see: third_party/ideep/mkl-dnn/LICENSE +Name: ms-gsl +License: MIT +Files: third_party/opentelemetry-cpp/third_party/ms-gsl + For details, see: third_party/opentelemetry-cpp/third_party/ms-gsl/LICENSE + Name: nccl License: BSD-3-Clause Files: third_party/nccl/nccl @@ -257,11 +328,6 @@ License: BSD-Source-Code Files: third_party/neon2sse For details, see: third_party/neon2sse/LICENSE -Name: oneDNN -License: Apache-2.0 -Files: third_party/ideep/mkl-dnn/third_party/oneDNN - For details, see: third_party/ideep/mkl-dnn/third_party/oneDNN/LICENSE - Name: onnx License: Apache-2.0 Files: third_party/onnx @@ -277,11 +343,46 @@ License: MIT Files: third_party/onnx-tensorrt For details, see: third_party/onnx-tensorrt/LICENSE +Name: opentelemetry-cpp +License: Apache-2.0 +Files: third_party/opentelemetry-cpp + For details, see: third_party/opentelemetry-cpp/LICENSE + +Name: opentelemetry-proto +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/third_party/opentelemetry-proto + For details, see: third_party/opentelemetry-cpp/third_party/opentelemetry-proto/LICENSE + +Name: opentracing-cpp +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp + For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/LICENSE + +Name: pdcurses +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/pdcurses + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/pdcurses/LICENSE + Name: pfs License: Apache-2.0 Files: third_party/kineto/libkineto/third_party/dynolog/third_party/pfs For details, see: third_party/kineto/libkineto/third_party/dynolog/third_party/pfs/LICENSE +Name: physac +License: MIT +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/physac + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/physac/LICENSE + +Name: pqp +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/pqp + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/pqp/LICENSE + +Name: prometheus-cpp +License: MIT +Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp + For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/LICENSE + Name: protobuf License: BSD-3-Clause Files: third_party/protobuf @@ -308,11 +409,21 @@ Files: third_party/onnx/third_party/pybind11, third_party/pybind11/LICENSE, third_party/tensorpipe/third_party/pybind11/LICENSE +Name: python +License: BSD-3-Clause +Files: third_party/cutlass/python + For details, see: third_party/cutlass/python/LICENSE.txt + Name: python-peachpy License: BSD-2-Clause Files: third_party/python-peachpy For details, see: third_party/python-peachpy/LICENSE.rst +Name: sigslot +License: Apache-2.0 +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/sigslot + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/sigslot/LICENSE + Name: sleef License: BSL-1.0 Files: third_party/sleef @@ -333,6 +444,11 @@ License: Apache-2.0 Files: third_party/tbb For details, see: third_party/tbb/LICENSE +Name: tensorflow-common +License: MIT +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/tensorflow-common + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/tensorflow-common/LICENSE.txt + Name: tensorpipe License: BSD-3-Clause Files: third_party/tensorpipe @@ -343,7 +459,22 @@ License: MIT with exception Files: third_party/kineto/libkineto/third_party/dynolog/third_party/cpr/test For details, see: third_party/kineto/libkineto/third_party/dynolog/third_party/cpr/test/LICENSE +Name: variant +License: BSD-3-Clause +Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/variant + For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/variant/LICENSE + +Name: vcpkg +License: MIT +Files: third_party/opentelemetry-cpp/tools/vcpkg + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/LICENSE.txt + +Name: vulkan +License: Apache-2.0 with exception +Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/vulkan + For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/vulkan/LICENSE.txt + Name: zstd License: BSD-3-Clause Files: third_party/zstd - For details, see: third_party/zstd/LICENSE + For details, see: third_party/zstd/LICENSE \ No newline at end of file diff --git a/third_party/build_bundled.py b/third_party/build_bundled.py index d60a2c1354fd..4e983400eb65 100644 --- a/third_party/build_bundled.py +++ b/third_party/build_bundled.py @@ -102,6 +102,21 @@ def identify_license(f, exception=''): elif 'BoostSoftwareLicense-Version1.0' in txt: # Hmm, do we need to check the text? return 'BSL-1.0' + elif 'gettimeofday' in txt: + # Used in opentelemetry-cpp/tools/vcpkg/ports/gettimeofday + return 'Apache-2.0' + elif 'libhungarian' in txt: + # Used in opentelemetry-cpp/tools/vcpkg/ports/hungarian + return 'Apache-2.0' + elif 'PDCurses' in txt: + # Used in opentelemetry-cpp/tools/vcpkg/ports/pdcurses + return 'Apache-2.0' + elif 'Copyright1999UniversityofNorthCarolina' in txt: + # Used in opentelemetry-cpp/tools/vcpkg/ports/pqp + return 'Apache-2.0' + elif 'sigslot' in txt: + # Used in opentelemetry-cpp/tools/vcpkg/ports/sigslot + return 'Apache-2.0' elif squeeze("Clarified Artistic License") in txt: return 'Clarified Artistic License' elif all([squeeze(m) in txt.lower() for m in bsd3_txt]): diff --git a/third_party/opentelemetry-cpp b/third_party/opentelemetry-cpp new file mode 160000 index 000000000000..a799f4aed9c9 --- /dev/null +++ b/third_party/opentelemetry-cpp @@ -0,0 +1 @@ +Subproject commit a799f4aed9c94b765dcdaabaeab7d5e7e2310878 diff --git a/third_party/opentelemetry-cpp.BUILD b/third_party/opentelemetry-cpp.BUILD new file mode 100644 index 000000000000..af58a8245290 --- /dev/null +++ b/third_party/opentelemetry-cpp.BUILD @@ -0,0 +1,71 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") + +package(default_visibility = ["//visibility:public"]) + +bool_flag( + name = "with_abseil", + build_setting_default = False, +) + +CPP_STDLIBS = [ + "none", + "best", + "2014", + "2017", + "2020", + "2023", +] + +string_flag( + name = "with_cxx_stdlib", + build_setting_default = "best", + values = CPP_STDLIBS, +) + +cc_library( + name = "api", + hdrs = glob(["include/**/*.h"]), + defines = select({ + ":with_external_abseil": ["HAVE_ABSEIL"], + "//conditions:default": [], + }) + select({ + ":set_cxx_stdlib_none": [], + ### automatic selection + ":set_cxx_stdlib_best": ["OPENTELEMETRY_STL_VERSION=(__cplusplus/100)"], + # See https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus + ":set_cxx_stdlib_best_and_msvc": ["OPENTELEMETRY_STL_VERSION=(_MSVC_LANG/100)"], + ### manual selection + ":set_cxx_stdlib_2014": ["OPENTELEMETRY_STL_VERSION=2014"], + ":set_cxx_stdlib_2017": ["OPENTELEMETRY_STL_VERSION=2017"], + ":set_cxx_stdlib_2020": ["OPENTELEMETRY_STL_VERSION=2020"], + ":set_cxx_stdlib_2023": ["OPENTELEMETRY_STL_VERSION=2023"], + "//conditions:default": [], + }), + strip_include_prefix = "include", + tags = ["api"], + deps = select({ + ":with_external_abseil": [ + "@com_google_absl//absl/base", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:variant", + ], + "//conditions:default": [], + }), +) + +config_setting( + name = "with_external_abseil", + flag_values = {":with_abseil": "true"}, +) + +[config_setting( + name = "set_cxx_stdlib_%s" % v, + flag_values = {":with_cxx_stdlib": v}, +) for v in CPP_STDLIBS] + +config_setting( + name = "set_cxx_stdlib_best_and_msvc", + constraint_values = ["@bazel_tools//tools/cpp:msvc"], + flag_values = {":with_cxx_stdlib": "best"}, +) diff --git a/torch/CMakeLists.txt b/torch/CMakeLists.txt index 97a72eed55bd..8f879a8ecc78 100644 --- a/torch/CMakeLists.txt +++ b/torch/CMakeLists.txt @@ -79,6 +79,7 @@ list(APPEND TORCH_PYTHON_INCLUDE_DIRECTORIES ${LIBSHM_SRCDIR}) set(TORCH_PYTHON_LINK_LIBRARIES python::python pybind::pybind11 + opentelemetry::api shm fmt::fmt-header-only ATEN_CPU_FILES_GEN_LIB)