create a concated LICENSE file for wheels (#81500)

Fixes #81181 by creating a temporary LICENCE file that has all the third-party licenses concatenated together when creating a wheel. Also update the `third_party/LICENSES_BUNDLED.txt` file.

The `third_party/LICENSES_BUNDLED.txt` file is supposed to be tested via `tests/test_license.py`, but the test is not running?
Pull Request resolved: https://github.com/pytorch/pytorch/pull/81500
Approved by: https://github.com/rgommers, https://github.com/seemethere
This commit is contained in:
mattip
2022-07-18 14:02:37 +00:00
committed by PyTorch MergeBot
parent d82d2083e2
commit 37474a54de
3 changed files with 122 additions and 159 deletions

View File

@ -664,20 +664,29 @@ class concat_license_files():
is a single license file in the sdist and wheels with all of the necessary
licensing info.
"""
def __init__(self):
def __init__(self, include_files=False):
self.f1 = 'LICENSE'
self.f2 = 'third_party/LICENSES_BUNDLED.txt'
self.include_files = include_files
def __enter__(self):
"""Concatenate files"""
old_path = sys.path
sys.path.append(third_party_path)
try:
from build_bundled import create_bundled
finally:
sys.path = old_path
with open(self.f1, 'r') as f1:
self.bsd_text = f1.read()
with open(self.f1, 'a') as f1:
with open(self.f2, 'r') as f2:
self.bundled_text = f2.read()
f1.write('\n\n')
f1.write(self.bundled_text)
f1.write('\n\n')
create_bundled(os.path.relpath(third_party_path), f1,
include_files=self.include_files)
def __exit__(self, exception_type, exception_value, traceback):
"""Restore content of f1"""
@ -697,7 +706,7 @@ else:
class wheel_concatenate(bdist_wheel):
""" check submodules on sdist to prevent incomplete tarballs """
def run(self):
with concat_license_files():
with concat_license_files(include_files=True):
super().run()
@ -937,9 +946,11 @@ def print_box(msg):
print('-' * (size + 2))
if __name__ == '__main__':
# Parse the command line and check the arguments
# before we proceed with building deps and setup
# Parse the command line and check the arguments before we proceed with
# building deps and setup. We need to set values so `--help` works.
dist = Distribution()
dist.script_name = os.path.basename(sys.argv[0])
dist.script_args = sys.argv[1:]
try:
dist.parse_command_line()
except setuptools.distutils.errors.DistutilsArgError as e:
@ -1158,7 +1169,7 @@ if __name__ == '__main__':
'Programming Language :: Python :: 3',
] + ['Programming Language :: Python :: 3.{}'.format(i) for i in range(python_min_version[1], version_range_max)],
license='BSD-3',
keywords='pytorch machine learning',
keywords='pytorch, machine learning',
)
if EMIT_BUILD_WARNING:
print_box(build_update_message)

232
third_party/LICENSES_BUNDLED.txt generated vendored
View File

@ -4,361 +4,301 @@ compatibly licensed. We list these here.
Name: FP16
License: MIT
Files: third_party/FP16
For details, see third_party/FP16/LICENSE
Name: FP16-source
License: MIT
Files: third_party/XNNPACK/build/FP16-source
For details, see third_party/XNNPACK/build/FP16-source/LICENSE
For details, see: third_party/FP16/LICENSE
Name: FXdiv
License: MIT
Files: third_party/FXdiv
For details, see third_party/FXdiv/LICENSE
Name: FXdiv-source
License: MIT
Files: third_party/XNNPACK/build/FXdiv-source
For details, see third_party/XNNPACK/build/FXdiv-source/LICENSE
For details, see: third_party/FXdiv/LICENSE
Name: NNPACK
License: BSD-2-Clause
Files: third_party/NNPACK
For details, see third_party/NNPACK/LICENSE
For details, see: third_party/NNPACK/LICENSE
Name: QNNPACK
License: BSD-3-Clause
Files: third_party/QNNPACK
For details, see third_party/QNNPACK/LICENSE
For details, see: third_party/QNNPACK/LICENSE
Name: XNNPACK
License: BSD-3-Clause
Files: third_party/XNNPACK
For details, see third_party/XNNPACK/LICENSE
For details, see: third_party/XNNPACK/LICENSE
Name: benchmark
License: Apache-2.0
Files: third_party/benchmark,
third_party/onnx/third_party/benchmark,
third_party/protobuf/third_party/benchmark,
third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark,
third_party/protobuf/third_party/benchmark
For details, see third_party/benchmark/LICENSE,
third_party/onnx/third_party/benchmark/LICENSE,
third_party/onnx/third_party/benchmark
For details, see: third_party/benchmark/LICENSE,
third_party/protobuf/third_party/benchmark/LICENSE,
third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark/LICENSE,
third_party/protobuf/third_party/benchmark/LICENSE
Name: breakpad
License: BSD-3-Clause
Files: third_party/breakpad
For details, see third_party/breakpad/LICENSE
third_party/onnx/third_party/benchmark/LICENSE
Name: clog
License: BSD-2-Clause
Files: third_party/QNNPACK/deps/clog,
third_party/XNNPACK/build/clog-source/deps/clog,
third_party/XNNPACK/build/cpuinfo-source/deps/clog,
third_party/cpuinfo/deps/clog,
third_party/fbgemm/third_party/cpuinfo/deps/clog
For details, see third_party/QNNPACK/deps/clog/LICENSE,
third_party/XNNPACK/build/clog-source/deps/clog/LICENSE,
third_party/XNNPACK/build/cpuinfo-source/deps/clog/LICENSE,
third_party/cpuinfo/deps/clog/LICENSE,
third_party/fbgemm/third_party/cpuinfo/deps/clog/LICENSE
Files: third_party/cpuinfo/deps/clog,
third_party/fbgemm/third_party/cpuinfo/deps/clog,
third_party/QNNPACK/deps/clog
For details, see: third_party/cpuinfo/deps/clog/LICENSE,
third_party/fbgemm/third_party/cpuinfo/deps/clog/LICENSE,
third_party/QNNPACK/deps/clog/LICENSE
Name: clog-source
License: BSD-2-Clause
Files: third_party/XNNPACK/build/clog-source
For details, see third_party/XNNPACK/build/clog-source/LICENSE
Name: cpplint
License: BSD-3-Clause
Files: third_party/nlohmann/tools/cpplint
For details, see: third_party/nlohmann/tools/cpplint/LICENSE
Name: cpuinfo
License: BSD-2-Clause
Files: third_party/cpuinfo,
third_party/fbgemm/third_party/cpuinfo
For details, see third_party/cpuinfo/LICENSE,
For details, see: third_party/cpuinfo/LICENSE,
third_party/fbgemm/third_party/cpuinfo/LICENSE
Name: cpuinfo-source
License: BSD-2-Clause
Files: third_party/XNNPACK/build/cpuinfo-source
For details, see third_party/XNNPACK/build/cpuinfo-source/LICENSE
Name: cudnn_frontend
License: MIT
Files: third_party/cudnn_frontend
For details, see third_party/cudnn_frontend/LICENSE.txt
For details, see: third_party/cudnn_frontend/LICENSE.txt
Name: dart
License: Apache-2.0
Files: third_party/flatbuffers/dart
For details, see third_party/flatbuffers/dart/LICENSE
For details, see: third_party/flatbuffers/dart/LICENSE
Name: doctest
License: MIT
Files: third_party/nlohmann/tests/thirdparty/doctest
For details, see: third_party/nlohmann/tests/thirdparty/doctest/LICENSE.txt
Name: eigen
License: BSD-3-Clause
Files: third_party/eigen
For details, see third_party/eigen/COPYING.BSD
For details, see: third_party/eigen/COPYING.BSD
Name: enum
License: BSD-3-Clause
Files: third_party/python-enum/enum
For details, see third_party/python-enum/enum/LICENSE
For details, see: third_party/python-enum/enum/LICENSE
Name: fbgemm
License: BSD-3-Clause
Files: third_party/fbgemm
For details, see third_party/fbgemm/LICENSE
For details, see: third_party/fbgemm/LICENSE
Name: flatbuffers
License: Apache-2.0
Files: third_party/flatbuffers
For details, see third_party/flatbuffers/LICENSE.txt
For details, see: third_party/flatbuffers/LICENSE.txt
Name: fmt
License: MIT with exception
Files: third_party/fmt,
third_party/kineto/libkineto/third_party/fmt
For details, see third_party/fmt/LICENSE.rst,
third_party/kineto/libkineto/third_party/fmt/LICENSE.rst
Files: third_party/kineto/libkineto/third_party/fmt,
third_party/fmt
For details, see: third_party/kineto/libkineto/third_party/fmt/LICENSE.rst,
third_party/fmt/LICENSE.rst
Name: foxi
License: MIT
Files: third_party/foxi
For details, see third_party/foxi/LICENSE
For details, see: third_party/foxi/LICENSE
Name: gemmlowp
License: Apache-2.0
Files: third_party/gemmlowp/gemmlowp
For details, see third_party/gemmlowp/gemmlowp/LICENSE
For details, see: third_party/gemmlowp/gemmlowp/LICENSE
Name: generator
License: Apache-2.0
Files: third_party/XNNPACK/build/googletest-source/googlemock/scripts/generator,
third_party/benchmark/build/third_party/googletest/src/googlemock/scripts/generator,
third_party/fbgemm/third_party/googletest/googlemock/scripts/generator,
Files: third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator,
third_party/googletest/googlemock/scripts/generator,
third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator,
third_party/fbgemm/third_party/googletest/googlemock/scripts/generator,
third_party/protobuf/third_party/googletest/googlemock/scripts/generator,
third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator
For details, see third_party/XNNPACK/build/googletest-source/googlemock/scripts/generator/LICENSE,
third_party/benchmark/build/third_party/googletest/src/googlemock/scripts/generator/LICENSE,
third_party/fbgemm/third_party/googletest/googlemock/scripts/generator/LICENSE,
For details, see: third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/kineto/libkineto/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/fbgemm/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/protobuf/third_party/googletest/googlemock/scripts/generator/LICENSE,
third_party/tensorpipe/third_party/googletest/googlemock/scripts/generator/LICENSE
Name: gloo
License: BSD-3-Clause
Files: third_party/gloo
For details, see third_party/gloo/LICENSE
Name: googlebenchmark-source
License: Apache-2.0
Files: third_party/XNNPACK/build/googlebenchmark-source
For details, see third_party/XNNPACK/build/googlebenchmark-source/LICENSE
For details, see: third_party/gloo/LICENSE
Name: googlemock
License: BSD-3-Clause
Files: third_party/XNNPACK/build/googletest-source/googlemock,
Files: third_party/kineto/libkineto/third_party/googletest/googlemock,
third_party/fbgemm/third_party/googletest/googlemock,
third_party/kineto/libkineto/third_party/googletest/googlemock,
third_party/protobuf/third_party/googletest/googlemock,
third_party/tensorpipe/third_party/googletest/googlemock
For details, see third_party/XNNPACK/build/googletest-source/googlemock/LICENSE,
For details, see: third_party/kineto/libkineto/third_party/googletest/googlemock/LICENSE,
third_party/fbgemm/third_party/googletest/googlemock/LICENSE,
third_party/kineto/libkineto/third_party/googletest/googlemock/LICENSE,
third_party/protobuf/third_party/googletest/googlemock/LICENSE,
third_party/tensorpipe/third_party/googletest/googlemock/LICENSE
Name: googletest
License: BSD-3-Clause
Files: third_party/XNNPACK/build/googletest-source/googletest,
Files: third_party/kineto/libkineto/third_party/googletest,
third_party/kineto/libkineto/third_party/googletest/googletest,
third_party/googletest,
third_party/fbgemm/third_party/googletest,
third_party/fbgemm/third_party/googletest/googletest,
third_party/googletest,
third_party/kineto/libkineto/third_party/googletest,
third_party/kineto/libkineto/third_party/googletest/googletest,
third_party/protobuf/third_party/googletest,
third_party/protobuf/third_party/googletest/googletest,
third_party/tensorpipe/third_party/googletest,
third_party/tensorpipe/third_party/googletest/googletest
For details, see third_party/XNNPACK/build/googletest-source/googletest/LICENSE,
For details, see: third_party/kineto/libkineto/third_party/googletest/LICENSE,
third_party/kineto/libkineto/third_party/googletest/googletest/LICENSE,
third_party/googletest/LICENSE,
third_party/fbgemm/third_party/googletest/LICENSE,
third_party/fbgemm/third_party/googletest/googletest/LICENSE,
third_party/googletest/LICENSE,
third_party/kineto/libkineto/third_party/googletest/LICENSE,
third_party/kineto/libkineto/third_party/googletest/googletest/LICENSE,
third_party/protobuf/third_party/googletest/LICENSE,
third_party/protobuf/third_party/googletest/googletest/LICENSE,
third_party/tensorpipe/third_party/googletest/LICENSE,
third_party/tensorpipe/third_party/googletest/googletest/LICENSE
Name: googletest-source
License: BSD-3-Clause
Files: third_party/XNNPACK/build/googletest-source
For details, see third_party/XNNPACK/build/googletest-source/LICENSE
Name: gtest
License: BSD-3-Clause
Files: third_party/ideep/mkl-dnn/tests/gtest,
third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest
For details, see third_party/ideep/mkl-dnn/tests/gtest/LICENSE,
For details, see: third_party/ideep/mkl-dnn/tests/gtest/LICENSE,
third_party/ideep/mkl-dnn/third_party/oneDNN/tests/gtests/gtest/LICENSE
Name: ideep
License: MIT
Files: third_party/ideep
For details, see third_party/ideep/LICENSE
For details, see: third_party/ideep/LICENSE
Name: ios-cmake
License: BSD-3-Clause
Files: third_party/ios-cmake
For details, see third_party/ios-cmake/LICENSE
For details, see: third_party/ios-cmake/LICENSE
Name: json
License: MIT
Files: third_party/cudnn_frontend/include/contrib/nlohmann/json
For details, see third_party/cudnn_frontend/include/contrib/nlohmann/json/LICENSE.txt
For details, see: third_party/cudnn_frontend/include/contrib/nlohmann/json/LICENSE.txt
Name: kineto
License: BSD-3-Clause
Files: third_party/kineto
For details, see third_party/kineto/LICENSE
Name: libdisasm
License: Clarified Artistic License
Files: third_party/breakpad/src/third_party/libdisasm
For details, see third_party/breakpad/src/third_party/libdisasm/LICENSE
For details, see: third_party/kineto/LICENSE
Name: libnop
License: Apache-2.0
Files: third_party/tensorpipe/third_party/libnop
For details, see third_party/tensorpipe/third_party/libnop/LICENSE
For details, see: third_party/tensorpipe/third_party/libnop/LICENSE
Name: libuv
License: MIT
Files: third_party/tensorpipe/third_party/libuv
For details, see third_party/tensorpipe/third_party/libuv/LICENSE
Name: lss
License: BSD-3-Clause
Files: third_party/breakpad/src/third_party/lss
For details, see third_party/breakpad/src/third_party/lss/LICENSE
For details, see: third_party/tensorpipe/third_party/libuv/LICENSE
Name: miniz-2.1.0
License: MIT
Files: third_party/miniz-2.1.0
For details, see third_party/miniz-2.1.0/LICENSE
For details, see: third_party/miniz-2.1.0/LICENSE
Name: mkl-dnn
License: Apache-2.0
Files: third_party/ideep/mkl-dnn
For details, see third_party/ideep/mkl-dnn/LICENSE
For details, see: third_party/ideep/mkl-dnn/LICENSE
Name: nccl
License: BSD-3-Clause
Files: third_party/nccl/nccl
For details, see third_party/nccl/nccl/LICENSE.txt
For details, see: third_party/nccl/nccl/LICENSE.txt
Name: neon2sse
License: BSD-Source-Code
Files: third_party/neon2sse
For details, see third_party/neon2sse/LICENSE
For details, see: third_party/neon2sse/LICENSE
Name: oneDNN
License: Apache-2.0
Files: third_party/ideep/mkl-dnn/third_party/oneDNN
For details, see third_party/ideep/mkl-dnn/third_party/oneDNN/LICENSE
Name: onnx
License: Apache-2.0
Files: third_party/onnx
For details, see third_party/onnx/LICENSE
For details, see: third_party/ideep/mkl-dnn/third_party/oneDNN/LICENSE
Name: onnx
License: MIT
Files: third_party/onnx-tensorrt/third_party/onnx
For details, see third_party/onnx-tensorrt/third_party/onnx/LICENSE
For details, see: third_party/onnx-tensorrt/third_party/onnx/LICENSE
Name: onnx
License: Apache-2.0
Files: third_party/onnx
For details, see: third_party/onnx/LICENSE
Name: onnx-tensorrt
License: MIT
Files: third_party/onnx-tensorrt
For details, see third_party/onnx-tensorrt/LICENSE
For details, see: third_party/onnx-tensorrt/LICENSE
Name: protobuf
License: BSD-3-Clause
Files: third_party/protobuf
For details, see third_party/protobuf/LICENSE
For details, see: third_party/protobuf/LICENSE
Name: psimd
License: MIT
Files: third_party/XNNPACK/deps/psimd,
third_party/psimd
For details, see third_party/XNNPACK/deps/psimd/LICENSE,
third_party/psimd/LICENSE
Files: third_party/psimd
For details, see: third_party/psimd/LICENSE
Name: pthreadpool
License: BSD-2-Clause
Files: third_party/pthreadpool
For details, see third_party/pthreadpool/LICENSE
Name: pthreadpool-source
License: BSD-2-Clause
Files: third_party/XNNPACK/build/pthreadpool-source
For details, see third_party/XNNPACK/build/pthreadpool-source/LICENSE
For details, see: third_party/pthreadpool/LICENSE
Name: pybind11
License: BSD-3-Clause
Files: third_party/onnx/third_party/pybind11,
Files: third_party/pybind11,
third_party/onnx-tensorrt/third_party/onnx/third_party/pybind11,
third_party/pybind11,
third_party/onnx/third_party/pybind11,
third_party/tensorpipe/third_party/pybind11
For details, see third_party/onnx/third_party/pybind11/LICENSE,
For details, see: third_party/pybind11/LICENSE,
third_party/onnx-tensorrt/third_party/onnx/third_party/pybind11/LICENSE,
third_party/pybind11/LICENSE,
third_party/onnx/third_party/pybind11/LICENSE,
third_party/tensorpipe/third_party/pybind11/LICENSE
Name: python-peachpy
License: BSD-2-Clause
Files: third_party/python-peachpy
For details, see third_party/python-peachpy/LICENSE.rst
For details, see: third_party/python-peachpy/LICENSE.rst
Name: python-six
License: MIT
Files: third_party/python-six
For details, see third_party/python-six/LICENSE
For details, see: third_party/python-six/LICENSE
Name: sleef
License: BSL-1.0
Files: third_party/sleef
For details, see third_party/sleef/LICENSE.txt
Name: src
License: BSD-3-Clause
Files: third_party/benchmark/build/third_party/googletest/src
For details, see third_party/benchmark/build/third_party/googletest/src/LICENSE
For details, see: third_party/sleef/LICENSE.txt
Name: swift
License: Apache-2.0
Files: third_party/flatbuffers/swift
For details, see third_party/flatbuffers/swift/LICENSE
For details, see: third_party/flatbuffers/swift/LICENSE
Name: tb_plugin
License: BSD-3-Clause
Files: third_party/kineto/tb_plugin
For details, see third_party/kineto/tb_plugin/LICENSE
For details, see: third_party/kineto/tb_plugin/LICENSE
Name: tbb
License: Apache-2.0
Files: third_party/tbb
For details, see third_party/tbb/LICENSE
For details, see: third_party/tbb/LICENSE
Name: tensorpipe
License: BSD-3-Clause
Files: third_party/tensorpipe
For details, see third_party/tensorpipe/LICENSE.txt
For details, see: third_party/tensorpipe/LICENSE.txt
Name: zstd
License: BSD-3-Clause
Files: third_party/zstd
For details, see third_party/zstd/LICENSE
For details, see: third_party/zstd/LICENSE

View File

@ -37,23 +37,35 @@ def collect_license(current):
return collected
def create_bundled(d, outstream):
def create_bundled(d, outstream, include_files=False):
"""Write the information to an open outstream"""
collected = collect_license(d)
sorted_keys = sorted(collected.keys())
outstream.write('The Pytorch repository and source distributions bundle '
'several libraries that are \n')
outstream.write('compatibly licensed. We list these here.\n\n')
outstream.write('compatibly licensed. We list these here.')
files_to_include = []
for k in sorted_keys:
c = collected[k]
files = ',\n '.join(c['Files'])
license_file = ',\n '.join(c['License_file'])
outstream.write('\n\n')
outstream.write(f"Name: {c['Name']}\n")
outstream.write(f"License: {c['License']}\n")
outstream.write(f"Files: {files}\n")
outstream.write(' For details, see ')
outstream.write(' For details, see')
if include_files:
outstream.write(' the files concatenated below: ')
files_to_include += c['License_file']
else:
outstream.write(': ')
outstream.write(license_file)
for fname in files_to_include:
outstream.write('\n\n')
outstream.write(fname)
outstream.write('\n' + '-' * len(fname) + '\n')
with open(fname, 'r') as fid:
outstream.write(fid.read())
def identify_license(f, exception=''):
@ -156,7 +168,7 @@ bsd3_src_txt = bsd3_txt[:2] + bsd3_txt[4:]
if __name__ == '__main__':
third_party = os.path.join(mydir)
third_party = os.path.relpath(mydir)
parser = argparse.ArgumentParser(
description="Generate bundled licenses file",
)