[rfc] opentelemetry in pytorch (#122999)

1. Add current latest version (opentelemetry-cpp version v1.14.2) to PyTorch library.
Steps:
```
$cd pytorch
$git submodule add https://github.com/open-telemetry/opentelemetry-cpp.git third_party/opentelemetry-cpp
$cd third_party/opentelemetry-cpp
$git checkout v1.14.2
$git add third_party/opentelemetry-cpp .gitmodules
$git commit
```
Expected change in checkout size:
```
(/home/cpio/local/a/pytorch-env) [cpio@devvm17556.vll0 ~/local/pytorch (gh/c-p-i-o/otel)]$ git count-objects -vH
count: 654
size: 3.59 MiB
in-pack: 1229701
packs: 17
size-pack: 1.17 GiB
prune-packable: 76
garbage: 0
size-garbage: 0 bytes
```

2.

TODO
- [x] Figure out how dynamic linking works. App builders will somehow need to `target_include` opentelemetry-cpp at runtime.
- [ ] Examples on how to use opentelemetry + pytorch
- [ ] Tests + documentation (e.g. using null opentelemetry implementation).

Pull Request resolved: https://github.com/pytorch/pytorch/pull/122999
Approved by: https://github.com/ezyang
This commit is contained in:
Chirag Pandya
2024-04-21 15:20:21 +00:00
committed by PyTorch MergeBot
parent 29cc293725
commit fd90991790
10 changed files with 287 additions and 22 deletions

View File

@ -1,3 +1,4 @@
# We do not use this library in our Bazel build. It contains an
# infinitely recursing symlink that makes Bazel very unhappy.
third_party/ittapi/
third_party/opentelemetry-cpp

3
.gitmodules vendored
View File

@ -149,3 +149,6 @@
[submodule "third_party/mimalloc"]
path = third_party/mimalloc
url = https://github.com/microsoft/mimalloc.git
[submodule "third_party/opentelemetry-cpp"]
path = third_party/opentelemetry-cpp
url = https://github.com/open-telemetry/opentelemetry-cpp.git

View File

@ -71,6 +71,13 @@ http_archive(
],
)
http_archive(
name = "com_github_opentelemetry-cpp",
urls = [
"https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.14.2.tar.gz",
],
)
new_local_repository(
name = "gloo",
build_file = "//third_party:gloo.BUILD",
@ -155,6 +162,12 @@ new_local_repository(
path = "third_party/kineto",
)
new_local_repository(
name = "opentelemetry-cpp",
build_file = "//third_party::opentelemetry-cpp.BUILD",
path = "third_party/opentelemetry-cpp",
)
new_patched_local_repository(
name = "tbb",
build_file = "//third_party:tbb.BUILD",

View File

@ -1156,6 +1156,16 @@ if(APPLE)
target_link_options(pybind::pybind11 INTERFACE -undefined dynamic_lookup)
endif()
# ---[ OpenTelemetry API headers
find_package(OpenTelemetryApi)
if(NOT OpenTelemetryApi_FOUND)
message(STATUS "Using third_party/opentelemetry-cpp.")
set(OpenTelemetryApi_INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/../third_party/opentelemetry-cpp/api/include)
endif()
message(STATUS "opentelemetry api include dirs: " "${OpenTelemetryApi_INCLUDE_DIRS}")
add_library(opentelemetry::api INTERFACE IMPORTED)
target_include_directories(opentelemetry::api SYSTEM INTERFACE ${OpenTelemetryApi_INCLUDE_DIRS})
# ---[ MPI
if(USE_MPI)
find_package(MPI)

View File

@ -0,0 +1,19 @@
# Try to find the OpenTelemetry API headers
# OpenTelemetryApi_FOUND - system has OpenTelemetry API headers
# OpenTelemetryApi_INCLUDE_DIRS - the OpenTelemetry API headers dir
find_path(OpenTelemetryApi_INCLUDE_DIR
NAMES include/opentelemetry/version.h
DOC "The directory where Open Telemetry API headers reside"
)
set(OpenTelemetryApi_INCLUDE_DIRS ${OpenTelemetryApi_INCLUDE_DIR})
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(OpenTelemetryApi
FOUND_VAR OpenTelemetryApi_FOUND
REQUIRED_VARS OpenTelemetryApi_INCLUDE_DIRS
)
mark_as_advanced(OpenTelemetryApi_FOUND)

175
third_party/LICENSES_BUNDLED.txt generated vendored
View File

@ -1,4 +1,4 @@
The Pytorch repository and source distributions bundle several libraries that are
The Pytorch repository and source distributions bundle several libraries that are
compatibly licensed. We list these here.
Name: DCGM
@ -41,11 +41,28 @@ License: Apache-2.0
Files: third_party/benchmark,
third_party/onnx/third_party/benchmark,
third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark,
third_party/protobuf/third_party/benchmark
third_party/protobuf/third_party/benchmark,
third_party/opentelemetry-cpp/third_party/benchmark
For details, see: third_party/benchmark/LICENSE,
third_party/onnx/third_party/benchmark/LICENSE,
third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark/LICENSE,
third_party/protobuf/third_party/benchmark/LICENSE
third_party/protobuf/third_party/benchmark/LICENSE,
third_party/opentelemetry-cpp/third_party/benchmark/LICENSE
Name: boost-vcpkg-helpers
License: MIT
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/boost-vcpkg-helpers
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/boost-vcpkg-helpers/LICENSE.txt
Name: cJSON
License: MIT
Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/examples/rest/cJSON
For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/examples/rest/cJSON/LICENSE
Name: catch2
License: BSL-1.0
Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/catch2
For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/catch2/LICENSE.txt
Name: clog
License: BSD-2-Clause
@ -104,6 +121,16 @@ Files: third_party/kineto/libkineto/third_party/dynolog/third_party/json/test/th
For details, see: third_party/kineto/libkineto/third_party/dynolog/third_party/json/test/thirdparty/doctest/LICENSE.txt,
third_party/nlohmann/tests/thirdparty/doctest/LICENSE.txt
Name: duktape-1.5.2
License: MIT
Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.5.2
For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.5.2/LICENSE.txt
Name: duktape-1.8.0
License: MIT
Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.8.0
For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/third_party/duktape-1.8.0/LICENSE.txt
Name: dynolog
License: MIT
Files: third_party/kineto/libkineto/third_party/dynolog
@ -114,22 +141,37 @@ License: BSD-3-Clause
Files: third_party/eigen
For details, see: third_party/eigen/COPYING.BSD
Name: etw
License: MIT
Files: third_party/opentelemetry-cpp/exporters/etw/include/opentelemetry/exporters/etw
For details, see: third_party/opentelemetry-cpp/exporters/etw/include/opentelemetry/exporters/etw/LICENSE
Name: expected
License: MIT
Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/expected
For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/expected/LICENSE
Name: fbgemm
License: BSD-3-Clause
Files: third_party/fbgemm
For details, see: third_party/fbgemm/LICENSE
Name: ffnvcodec
License: MIT with exception
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/ffnvcodec
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/ffnvcodec/LICENSE.txt
Name: flatbuffers
License: Apache-2.0
Files: third_party/flatbuffers
For details, see: third_party/flatbuffers/LICENSE.txt
For details, see: third_party/flatbuffers/LICENSE
Name: fmt
License: MIT with exception
Files: third_party/fmt,
third_party/kineto/libkineto/third_party/dynolog/third_party/fmt,
third_party/kineto/libkineto/third_party/fmt
For details, see: third_party/fmt/LICENSE.rst,
For details, see: third_party/fmt/LICENSE,
third_party/kineto/libkineto/third_party/dynolog/third_party/fmt/LICENSE.rst,
third_party/kineto/libkineto/third_party/fmt/LICENSE.rst
@ -149,12 +191,19 @@ Files: third_party/fbgemm/third_party/googletest/googlemock/scripts/generator,
third_party/googletest/googlemock/scripts/generator,
third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator,
third_party/protobuf/third_party/googletest/googlemock/scripts/generator,
third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator
third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator,
third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest/googlemock/scripts/generator
For details, see: third_party/fbgemm/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/protobuf/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator/LICENSE
third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest/googlemock/scripts/generator/LICENSE
Name: gettimeofday
License: Apache-2.0
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/gettimeofday
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/gettimeofday/LICENSE
Name: gloo
License: BSD-3-Clause
@ -183,7 +232,9 @@ Files: third_party/fbgemm/third_party/googletest,
third_party/protobuf/third_party/googletest,
third_party/protobuf/third_party/googletest/googletest,
third_party/tensorpipe/third_party/googletest,
third_party/tensorpipe/third_party/googletest/googletest
third_party/tensorpipe/third_party/googletest/googletest,
third_party/opentelemetry-cpp/third_party/googletest,
third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest
For details, see: third_party/fbgemm/third_party/googletest/LICENSE,
third_party/fbgemm/third_party/googletest/googletest/LICENSE,
third_party/googletest/LICENSE,
@ -193,20 +244,25 @@ Files: third_party/fbgemm/third_party/googletest,
third_party/protobuf/third_party/googletest/LICENSE,
third_party/protobuf/third_party/googletest/googletest/LICENSE,
third_party/tensorpipe/third_party/googletest/LICENSE,
third_party/tensorpipe/third_party/googletest/googletest/LICENSE
third_party/tensorpipe/third_party/googletest/googletest/LICENSE,
third_party/opentelemetry-cpp/third_party/googletest/LICENSE,
third_party/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/googletest/LICENSE
Name: gtest
License: BSD-3-Clause
Files: third_party/ideep/mkl-dnn/tests/gtest,
third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest
For details, see: third_party/ideep/mkl-dnn/tests/gtest/LICENSE,
third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest/LICENSE
Files: third_party/ideep/mkl-dnn/tests/gtests/gtest
For details, see: third_party/ideep/mkl-dnn/tests/gtests/gtest/LICENSE
Name: hipify_torch
License: MIT
Files: third_party/fbgemm/third_party/hipify_torch
For details, see: third_party/fbgemm/third_party/hipify_torch/LICENSE.txt
Name: hungarian
License: Apache-2.0
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/hungarian
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/hungarian/LICENSE.txt
Name: ideep
License: MIT
Files: third_party/ideep
@ -217,10 +273,10 @@ License: BSD-3-Clause
Files: third_party/ios-cmake
For details, see: third_party/ios-cmake/LICENSE
Name: json
Name: irrlicht
License: MIT
Files: third_party/cudnn_frontend/include/contrib/nlohmann/json
For details, see: third_party/cudnn_frontend/include/contrib/nlohmann/json/LICENSE.txt
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/irrlicht
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/irrlicht/LICENSE.txt
Name: kineto
License: BSD-3-Clause
@ -232,11 +288,21 @@ License: Apache-2.0
Files: third_party/tensorpipe/third_party/libnop
For details, see: third_party/tensorpipe/third_party/libnop/LICENSE
Name: libstemmer
License: BSD-3-Clause
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/libstemmer
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/libstemmer/LICENSE
Name: libuv
License: MIT
Files: third_party/tensorpipe/third_party/libuv
For details, see: third_party/tensorpipe/third_party/libuv/LICENSE
Name: mimalloc
License: MIT
Files: third_party/mimalloc
For details, see: third_party/mimalloc/LICENSE
Name: miniz-2.1.0
License: MIT
Files: third_party/miniz-2.1.0
@ -247,6 +313,11 @@ License: Apache-2.0
Files: third_party/ideep/mkl-dnn
For details, see: third_party/ideep/mkl-dnn/LICENSE
Name: ms-gsl
License: MIT
Files: third_party/opentelemetry-cpp/third_party/ms-gsl
For details, see: third_party/opentelemetry-cpp/third_party/ms-gsl/LICENSE
Name: nccl
License: BSD-3-Clause
Files: third_party/nccl/nccl
@ -257,11 +328,6 @@ License: BSD-Source-Code
Files: third_party/neon2sse
For details, see: third_party/neon2sse/LICENSE
Name: oneDNN
License: Apache-2.0
Files: third_party/ideep/mkl-dnn/third_party/oneDNN
For details, see: third_party/ideep/mkl-dnn/third_party/oneDNN/LICENSE
Name: onnx
License: Apache-2.0
Files: third_party/onnx
@ -277,11 +343,46 @@ License: MIT
Files: third_party/onnx-tensorrt
For details, see: third_party/onnx-tensorrt/LICENSE
Name: opentelemetry-cpp
License: Apache-2.0
Files: third_party/opentelemetry-cpp
For details, see: third_party/opentelemetry-cpp/LICENSE
Name: opentelemetry-proto
License: Apache-2.0
Files: third_party/opentelemetry-cpp/third_party/opentelemetry-proto
For details, see: third_party/opentelemetry-cpp/third_party/opentelemetry-proto/LICENSE
Name: opentracing-cpp
License: Apache-2.0
Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp
For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/LICENSE
Name: pdcurses
License: Apache-2.0
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/pdcurses
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/pdcurses/LICENSE
Name: pfs
License: Apache-2.0
Files: third_party/kineto/libkineto/third_party/dynolog/third_party/pfs
For details, see: third_party/kineto/libkineto/third_party/dynolog/third_party/pfs/LICENSE
Name: physac
License: MIT
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/physac
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/physac/LICENSE
Name: pqp
License: Apache-2.0
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/pqp
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/pqp/LICENSE
Name: prometheus-cpp
License: MIT
Files: third_party/opentelemetry-cpp/third_party/prometheus-cpp
For details, see: third_party/opentelemetry-cpp/third_party/prometheus-cpp/LICENSE
Name: protobuf
License: BSD-3-Clause
Files: third_party/protobuf
@ -308,11 +409,21 @@ Files: third_party/onnx/third_party/pybind11,
third_party/pybind11/LICENSE,
third_party/tensorpipe/third_party/pybind11/LICENSE
Name: python
License: BSD-3-Clause
Files: third_party/cutlass/python
For details, see: third_party/cutlass/python/LICENSE.txt
Name: python-peachpy
License: BSD-2-Clause
Files: third_party/python-peachpy
For details, see: third_party/python-peachpy/LICENSE.rst
Name: sigslot
License: Apache-2.0
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/sigslot
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/sigslot/LICENSE
Name: sleef
License: BSL-1.0
Files: third_party/sleef
@ -333,6 +444,11 @@ License: Apache-2.0
Files: third_party/tbb
For details, see: third_party/tbb/LICENSE
Name: tensorflow-common
License: MIT
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/tensorflow-common
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/tensorflow-common/LICENSE.txt
Name: tensorpipe
License: BSD-3-Clause
Files: third_party/tensorpipe
@ -343,7 +459,22 @@ License: MIT with exception
Files: third_party/kineto/libkineto/third_party/dynolog/third_party/cpr/test
For details, see: third_party/kineto/libkineto/third_party/dynolog/third_party/cpr/test/LICENSE
Name: variant
License: BSD-3-Clause
Files: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/variant
For details, see: third_party/opentelemetry-cpp/third_party/opentracing-cpp/3rd_party/include/opentracing/variant/LICENSE
Name: vcpkg
License: MIT
Files: third_party/opentelemetry-cpp/tools/vcpkg
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/LICENSE.txt
Name: vulkan
License: Apache-2.0 with exception
Files: third_party/opentelemetry-cpp/tools/vcpkg/ports/vulkan
For details, see: third_party/opentelemetry-cpp/tools/vcpkg/ports/vulkan/LICENSE.txt
Name: zstd
License: BSD-3-Clause
Files: third_party/zstd
For details, see: third_party/zstd/LICENSE
For details, see: third_party/zstd/LICENSE

View File

@ -102,6 +102,21 @@ def identify_license(f, exception=''):
elif 'BoostSoftwareLicense-Version1.0' in txt:
# Hmm, do we need to check the text?
return 'BSL-1.0'
elif 'gettimeofday' in txt:
# Used in opentelemetry-cpp/tools/vcpkg/ports/gettimeofday
return 'Apache-2.0'
elif 'libhungarian' in txt:
# Used in opentelemetry-cpp/tools/vcpkg/ports/hungarian
return 'Apache-2.0'
elif 'PDCurses' in txt:
# Used in opentelemetry-cpp/tools/vcpkg/ports/pdcurses
return 'Apache-2.0'
elif 'Copyright1999UniversityofNorthCarolina' in txt:
# Used in opentelemetry-cpp/tools/vcpkg/ports/pqp
return 'Apache-2.0'
elif 'sigslot' in txt:
# Used in opentelemetry-cpp/tools/vcpkg/ports/sigslot
return 'Apache-2.0'
elif squeeze("Clarified Artistic License") in txt:
return 'Clarified Artistic License'
elif all([squeeze(m) in txt.lower() for m in bsd3_txt]):

71
third_party/opentelemetry-cpp.BUILD vendored Normal file
View File

@ -0,0 +1,71 @@
load("@rules_cc//cc:defs.bzl", "cc_library")
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
package(default_visibility = ["//visibility:public"])
bool_flag(
name = "with_abseil",
build_setting_default = False,
)
CPP_STDLIBS = [
"none",
"best",
"2014",
"2017",
"2020",
"2023",
]
string_flag(
name = "with_cxx_stdlib",
build_setting_default = "best",
values = CPP_STDLIBS,
)
cc_library(
name = "api",
hdrs = glob(["include/**/*.h"]),
defines = select({
":with_external_abseil": ["HAVE_ABSEIL"],
"//conditions:default": [],
}) + select({
":set_cxx_stdlib_none": [],
### automatic selection
":set_cxx_stdlib_best": ["OPENTELEMETRY_STL_VERSION=(__cplusplus/100)"],
# See https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus
":set_cxx_stdlib_best_and_msvc": ["OPENTELEMETRY_STL_VERSION=(_MSVC_LANG/100)"],
### manual selection
":set_cxx_stdlib_2014": ["OPENTELEMETRY_STL_VERSION=2014"],
":set_cxx_stdlib_2017": ["OPENTELEMETRY_STL_VERSION=2017"],
":set_cxx_stdlib_2020": ["OPENTELEMETRY_STL_VERSION=2020"],
":set_cxx_stdlib_2023": ["OPENTELEMETRY_STL_VERSION=2023"],
"//conditions:default": [],
}),
strip_include_prefix = "include",
tags = ["api"],
deps = select({
":with_external_abseil": [
"@com_google_absl//absl/base",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/types:variant",
],
"//conditions:default": [],
}),
)
config_setting(
name = "with_external_abseil",
flag_values = {":with_abseil": "true"},
)
[config_setting(
name = "set_cxx_stdlib_%s" % v,
flag_values = {":with_cxx_stdlib": v},
) for v in CPP_STDLIBS]
config_setting(
name = "set_cxx_stdlib_best_and_msvc",
constraint_values = ["@bazel_tools//tools/cpp:msvc"],
flag_values = {":with_cxx_stdlib": "best"},
)

View File

@ -79,6 +79,7 @@ list(APPEND TORCH_PYTHON_INCLUDE_DIRECTORIES ${LIBSHM_SRCDIR})
set(TORCH_PYTHON_LINK_LIBRARIES
python::python
pybind::pybind11
opentelemetry::api
shm
fmt::fmt-header-only
ATEN_CPU_FILES_GEN_LIB)