mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
Hi! I've been fuzzing different pytorch modules with with [sydr-fuzz](https://github.com/ispras/oss-sydr-fuzz/tree/master/projects/pytorch), and found a multiple crashes in torch::jit::load() function. All found errors could be reproduced with provided docker: [Dockerfile](https://github.com/ispras/oss-sydr-fuzz/tree/master/projects/pytorch). ### Crash in torch/csrc/jit/unpickler.cpp:1075 [crash-1f59083b8396c5b62b4705c7556e68f129e833b1.zip](https://github.com/pytorch/pytorch/files/11552947/crash-1f59083b8396c5b62b4705c7556e68f129e833b1.zip) ```asan "#0 0x00007ffff7a5600b in raise () from /lib/x86_64-linux-gnu/libc.so.6", "#1 0x00007ffff7a35859 in abort () from /lib/x86_64-linux-gnu/libc.so.6", "#2 0x00007ffff7ce3911 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#3 0x00007ffff7cef38c in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#4 0x00007ffff7cef3f7 in std::terminate() () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#5 0x00007ffff7cef6a9 in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#6 0x00007ffff7ce6326 in std::__throw_length_error(char const*) () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#7 0x00007ffff7d87edc in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_create(unsigned long&, unsigned long) () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#8 0x00007ffff7d88880 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::reserve(unsigned long) () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#9 0x000000000ea52931 in torch::jit::Unpickler::readBytes[abi:cxx11](unsigned long) (this=this@entry=0x7fffffffac10, length=length@entry=8358680908539635837) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:1075", "#10 0x000000000ea4c3a0 in torch::jit::Unpickler::readInstruction (this=0x7fffffff90d0) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:355", "#11 0x000000000ea49eb8 in torch::jit::Unpickler::run (this=0x7fffffffac10) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:251", "#12 0x000000000ea49b12 in torch::jit::Unpickler::parse_ivalue (this=0x7fffffffac10) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:204", "#13 0x000000000e960a9f in torch::jit::readArchiveAndTensors(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::optional<std::function<c10::StrongTypePtr (c10::QualifiedName const&)> >, c10::optional<std::function<c10::intrusive_ptr<c10::ivalue::Object, c10::detail::intrusive_target_default_null_type<c10::ivalue::Object> > (c10::StrongTypePtr, c10::IValue)> >, c10::optional<c10::Device>, caffe2::serialize::PyTorchStreamReader&, c10::Type::SingletonOrSharedTypePtr<c10::Type> (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), std::shared_ptr<torch::jit::DeserializationStorageContext>) (archive_name=..., pickle_prefix=..., tensor_prefix=..., type_resolver=..., obj_loader=..., device=..., stream_reader=..., type_parser=<optimized out>, storage_context=...) at /pytorch/torch/csrc/jit/serialization/import_read.cpp:53", "#14 0x000000000e8ef599 in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive (this=0x7fffffffbc60, archive_name=...) at /pytorch/torch/csrc/jit/serialization/import.cpp:184", "#15 0x000000000e8eb886 in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::deserialize (this=<optimized out>, device=..., extra_files=..., restore_shapes=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:287", "#16 0x000000000e8e9cc5 in torch::jit::import_ir_module (cu=..., in=..., device=..., extra_files=..., load_debug_files=<optimized out>, restore_shapes=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:386", "#17 0x000000000e8f37bf in torch::jit::import_ir_module (cu=..., in=..., device=..., load_debug_files=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:322", "#18 0x000000000e8f615a in torch::jit::load (in=..., device=..., load_debug_files=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:482", "#19 0x00000000005c2d61 in LLVMFuzzerTestOneInput (data=<optimized out>, size=1663) at /load.cc:42", "#20 0x00000000005c2a8e in ExecuteFilesOnyByOne (argc=2, argv=0x7fffffffc6b8, callback=callback@entry=0x5c2ae0 <LLVMFuzzerTestOneInput(uint8_t const*, size_t)>) at /AFLplusplus/utils/aflpp_driver/aflpp_driver.c:255", "#21 0x00000000005c2899 in LLVMFuzzerRunDriver (argcp=argcp@entry=0x7fffffffc5b4, argvp=argvp@entry=0x7fffffffc5b8, callback=0x5c2ae0 <LLVMFuzzerTestOneInput(uint8_t const*, size_t)>) at /AFLplusplus/utils/aflpp_driver/aflpp_driver.c:364", "#22 0x00000000005c2459 in main (argc=2, argv=0x7fffffffc6b8) at /AFLplusplus/utils/aflpp_driver/aflpp_driver.c:300" ``` ### Crash in torch/csrc/jit/unpickler.cpp:386 [crash-2e9923de375c393e700e8c0441f0ebe8252ca364.zip](https://github.com/pytorch/pytorch/files/11552950/crash-2e9923de375c393e700e8c0441f0ebe8252ca364.zip) ```asan "#0 0x00007ffff7a5600b in raise () from /lib/x86_64-linux-gnu/libc.so.6", "#1 0x00007ffff7a35859 in abort () from /lib/x86_64-linux-gnu/libc.so.6", "#2 0x00007ffff7ce3911 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#3 0x00007ffff7cef38c in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#4 0x00007ffff7cef3f7 in std::terminate() () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#5 0x00007ffff7cef6a9 in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#6 0x00007ffff7ce6326 in std::__throw_length_error(char const*) () from /lib/x86_64-linux-gnu/libstdc++.so.6", "#7 0x0000000000670aff in std::vector<c10::IValue, std::allocator<c10::IValue> >::reserve (this=this@entry=0x7fffffff9750, __n=__n@entry=18446744073709551614) at /usr/include/c++/10/bits/vector.tcc:70", "#8 0x000000000ea4d5cd in torch::jit::Unpickler::readInstruction (this=0x7fffffffac10) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:386", "#9 0x000000000ea49eb8 in torch::jit::Unpickler::run (this=0x7fffffffac10) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:251", "#10 0x000000000ea49b12 in torch::jit::Unpickler::parse_ivalue (this=0x7fffffffac10) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:204", "#11 0x000000000e960a9f in torch::jit::readArchiveAndTensors(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::optional<std::function<c10::StrongTypePtr (c10::QualifiedName const&)> >, c10::optional<std::function<c10::intrusive_ptr<c10::ivalue::Object, c10::detail::intrusive_target_default_null_type<c10::ivalue::Object> > (c10::StrongTypePtr, c10::IValue)> >, c10::optional<c10::Device>, caffe2::serialize::PyTorchStreamReader&, c10::Type::SingletonOrSharedTypePtr<c10::Type> (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), std::shared_ptr<torch::jit::DeserializationStorageContext>) (archive_name=..., pickle_prefix=..., tensor_prefix=..., type_resolver=..., obj_loader=..., device=..., stream_reader=..., type_parser=<optimized out>, storage_context=...) at /pytorch/torch/csrc/jit/serialization/import_read.cpp:53", "#12 0x000000000e8ef599 in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive (this=0x7fffffffbc60, archive_name=...) at /pytorch/torch/csrc/jit/serialization/import.cpp:184", "#13 0x000000000e8eb886 in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::deserialize (this=<optimized out>, device=..., extra_files=..., restore_shapes=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:287", "#14 0x000000000e8e9cc5 in torch::jit::import_ir_module (cu=..., in=..., device=..., extra_files=..., load_debug_files=<optimized out>, restore_shapes=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:386", "#15 0x000000000e8f37bf in torch::jit::import_ir_module (cu=..., in=..., device=..., load_debug_files=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:322", "#16 0x000000000e8f615a in torch::jit::load (in=..., device=..., load_debug_files=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:482", "#17 0x00000000005c2d61 in LLVMFuzzerTestOneInput (data=<optimized out>, size=5498) at /load.cc:42", "#18 0x00000000005c2a8e in ExecuteFilesOnyByOne (argc=2, argv=0x7fffffffc6b8, callback=callback@entry=0x5c2ae0 <LLVMFuzzerTestOneInput(uint8_t const*, size_t)>) at /AFLplusplus/utils/aflpp_driver/aflpp_driver.c:255", "#19 0x00000000005c2899 in LLVMFuzzerRunDriver (argcp=argcp@entry=0x7fffffffc5b4, argvp=argvp@entry=0x7fffffffc5b8, callback=0x5c2ae0 <LLVMFuzzerTestOneInput(uint8_t const*, size_t)>) at /AFLplusplus/utils/aflpp_driver/aflpp_driver.c:364", "#20 0x00000000005c2459 in main (argc=2, argv=0x7fffffffc6b8) at /AFLplusplus/utils/aflpp_driver/aflpp_driver.c:300" ``` ### Crash in torch/csrc/jit/serialization/source_range_serialization.cpp:211 [crash-5598d386057152f606bfa69d85605499e8852625.zip](https://github.com/pytorch/pytorch/files/11552952/crash-5598d386057152f606bfa69d85605499e8852625.zip) ```asan "#0 torch::jit::ConcreteSourceRangeUnpickler::unpickle (this=0x99b8d80) at /pytorch/torch/csrc/jit/serialization/source_range_serialization.cpp:211", "#1 0x0000000004042566 in torch::jit::ConcreteSourceRangeUnpickler::findSourceRangeThatGenerated (this=0x99aa1c0, range=...) at /pytorch/torch/csrc/jit/serialization/source_range_serialization.cpp:229", "#2 0x00000000007b5cc8 in torch::jit::Source::findSourceRangeThatGenerated (this=<optimized out>, range=...) at /pytorch/torch/csrc/jit/frontend/source_range.cpp:144", "#3 torch::jit::SourceRange::findSourceRangeThatGenerated (this=0x7fffffffa650) at /pytorch/torch/csrc/jit/frontend/source_range.h:384", "#4 torch::jit::SourceRange::highlight (this=0x7fffffffa650, out=...) at /pytorch/torch/csrc/jit/frontend/source_range.cpp:149", "#5 0x00000000007a0e74 in torch::jit::Lexer::expected (this=this@entry=0x99979a0, what=..., t=...) at /pytorch/torch/csrc/jit/frontend/lexer.h:461", "#6 0x000000000079fcaa in torch::jit::Lexer::lexRaw (this=this@entry=0x99979a0, whitespace_token=false) at /pytorch/torch/csrc/jit/frontend/lexer.h:552", "#7 0x000000000079fd23 in torch::jit::Lexer::lex (this=this@entry=0x99979a0) at /pytorch/torch/csrc/jit/frontend/lexer.h:487", "#8 0x00000000007a1da1 in torch::jit::Lexer::next (this=this@entry=0x99979a0) at /pytorch/torch/csrc/jit/frontend/lexer.h:436", "#9 0x0000000003bff6a8 in torch::jit::Lexer::nextIf (this=0x99979a0, kind=330) at /pytorch/torch/csrc/jit/frontend/lexer.h:444", "#10 torch::jit::ParserImpl::parseReturnAnnotation (this=this@entry=0x99979a0) at /pytorch/torch/csrc/jit/frontend/parser.cpp:703", "#11 0x0000000003bfd500 in torch::jit::ParserImpl::parseDecl (this=this@entry=0x99979a0) at /pytorch/torch/csrc/jit/frontend/parser.cpp:729", "#12 0x0000000003bfb725 in torch::jit::ParserImpl::parseFunction (this=this@entry=0x99979a0, is_method=true) at /pytorch/torch/csrc/jit/frontend/parser.cpp:755", "#13 0x0000000003bfdc28 in torch::jit::ParserImpl::parseStmt (this=this@entry=0x99979a0, in_class=<optimized out>) at /pytorch/torch/csrc/jit/frontend/parser.cpp:599", "#14 0x0000000003bfd8dd in torch::jit::ParserImpl::parseStatements (this=this@entry=0x99979a0, expect_indent=<optimized out>, in_class=<optimized out>) at /pytorch/torch/csrc/jit/frontend/parser.cpp:697", "#15 0x0000000003bfc4ba in torch::jit::ParserImpl::parseClass (this=0x99979a0) at /pytorch/torch/csrc/jit/frontend/parser.cpp:747", "#16 0x0000000003bfaddc in torch::jit::Parser::parseClass (this=<optimized out>) at /pytorch/torch/csrc/jit/frontend/parser.cpp:812", "#17 0x0000000004008e2d in torch::jit::SourceImporterImpl::parseSourceIfNeeded (this=this@entry=0x95d41f0, qualifier=...) at /pytorch/torch/csrc/jit/serialization/import_source.cpp:182", "#18 0x0000000004008ab7 in torch::jit::SourceImporterImpl::findNamedType (this=this@entry=0x95d41f0, name=...) at /pytorch/torch/csrc/jit/serialization/import_source.cpp:135", "#19 0x000000000400d010 in torch::jit::SourceImporterImpl::resolveType (this=0x95d41f0, name=..., loc=...) at /pytorch/torch/csrc/jit/serialization/import_source.cpp:261", "#20 0x0000000003c20821 in torch::jit::ScriptTypeParser::parseTypeFromExpr (this=this@entry=0x7fffffffb658, expr=...) at /pytorch/torch/csrc/jit/frontend/script_type_parser.cpp:238", "#21 0x0000000003c20acc in torch::jit::ScriptTypeParser::parseType (this=0x7fffffffb658, str=...) at /pytorch/torch/csrc/jit/frontend/script_type_parser.cpp:312", "#22 0x0000000004019416 in torch::jit::SourceImporter::loadType (this=<optimized out>, name=...) at /pytorch/torch/csrc/jit/serialization/import_source.cpp:786", "#23 0x0000000003ff365e in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::$_0::operator()(c10::QualifiedName const&) const (this=<optimized out>, qn=...) at /pytorch/torch/csrc/jit/serialization/import.cpp:146", "#24 std::__invoke_impl<c10::StrongTypePtr, torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::$_0&, c10::QualifiedName const&>(std::__invoke_other, torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::$_0&, c10::QualifiedName const&) (__f=..., __args=...) at /usr/include/c++/10/bits/invoke.h:60", "#25 std::__invoke_r<c10::StrongTypePtr, torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::$_0&, c10::QualifiedName const&>(torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::$_0&, c10::QualifiedName const&) (__fn=..., __args=...) at /usr/include/c++/10/bits/invoke.h:113", "#26 std::_Function_handler<c10::StrongTypePtr (c10::QualifiedName const&), torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::$_0>::_M_invoke(std::_Any_data const&, c10::QualifiedName const&) (__functor=..., __args=...) at /usr/include/c++/10/bits/std_function.h:291", "#27 0x000000000404e5c4 in std::function<c10::StrongTypePtr (c10::QualifiedName const&)>::operator()(c10::QualifiedName const&) const (this=0x7fffffffbf28, __args=...) at /usr/include/c++/10/bits/std_function.h:622", "#28 torch::jit::Unpickler::readGlobal (this=this@entry=0x7fffffffbd50, module_name=..., class_name=...) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:820", "#29 0x0000000004049ce5 in torch::jit::Unpickler::readInstruction (this=this@entry=0x7fffffffbd50) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:496", "#30 0x00000000040497a8 in torch::jit::Unpickler::run (this=0x7fffffffbd50) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:251", "#31 0x00000000040494f9 in torch::jit::Unpickler::parse_ivalue (this=0x99aa1c0) at /pytorch/torch/csrc/jit/serialization/unpickler.cpp:204", "#32 0x00000000040075f8 in torch::jit::readArchiveAndTensors(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::optional<std::function<c10::StrongTypePtr (c10::QualifiedName const&)> >, c10::optional<std::function<c10::intrusive_ptr<c10::ivalue::Object, c10::detail::intrusive_target_default_null_type<c10::ivalue::Object> > (c10::StrongTypePtr, c10::IValue)> >, c10::optional<c10::Device>, caffe2::serialize::PyTorchStreamReader&, c10::Type::SingletonOrSharedTypePtr<c10::Type> (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), std::shared_ptr<torch::jit::DeserializationStorageContext>) (archive_name=..., pickle_prefix=..., tensor_prefix=..., type_resolver=..., obj_loader=..., device=..., stream_reader=..., type_parser=0x0, storage_context=...) at /pytorch/torch/csrc/jit/serialization/import_read.cpp:53", "#33 0x0000000003ff3545 in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::readArchive (this=this@entry=0x7fffffffc2b8, archive_name=...) at /pytorch/torch/csrc/jit/serialization/import.cpp:184", "#34 0x0000000003fed8bf in torch::jit::(anonymous namespace)::ScriptModuleDeserializer::deserialize (this=this@entry=0x7fffffffc2b8, device=device@entry=..., extra_files=..., restore_shapes=220) at /pytorch/torch/csrc/jit/serialization/import.cpp:287", "#35 0x0000000003febb0f in torch::jit::import_ir_module (cu=..., in=..., device=..., device@entry=..., extra_files=..., load_debug_files=true, restore_shapes=<optimized out>) at /pytorch/torch/csrc/jit/serialization/import.cpp:386", "#36 0x0000000003feb7a1 in torch::jit::import_ir_module (cu=..., in=..., device=..., device@entry=..., load_debug_files=false) at /pytorch/torch/csrc/jit/serialization/import.cpp:322", "#37 0x0000000003ff015a in torch::jit::load (in=..., device=device@entry=..., load_debug_files=true) at /pytorch/torch/csrc/jit/serialization/import.cpp:482", "#38 0x00000000004a1655 in LLVMFuzzerTestOneInput (data=0x981a680 \"PK\\003\\004\", size=1609) at /load.cc:42", "#39 0x00000000004a1dbf in main ()" ``` ### Segmentation fault in /pytorch/aten/src/ATen/core/ivalue.h:526 [crash-9bd059c1ae85ab9cdb41d786932214d942baa189.zip](https://github.com/pytorch/pytorch/files/11552956/crash-9bd059c1ae85ab9cdb41d786932214d942baa189.zip) ```asan "==8528==ERROR: AddressSanitizer: SEGV on unknown address (pc 0x00000e55d97e bp 0x7fffffffb4d0 sp 0x7fffffffb360 T0)", "==8528==The signal is caused by a READ memory access.", "==8528==Hint: this fault was caused by a dereference of a high value address (see register values below). Disassemble the provided pc to learn which register was used.", " #0 0xe55d97e in c10::IValue::isTuple() const /pytorch/aten/src/ATen/core/ivalue.h:526:26", " #1 0xe55d97e in torch::distributed::rpc::GloballyUniqueId::fromIValue(c10::IValue const&) /pytorch/torch/csrc/distributed/rpc/types.cpp:60:3", " #2 0xe4b04fb in torch::distributed::rpc::ScriptRemoteCall::fromIValues(std::vector<c10::IValue, std::allocator<c10::IValue> >&) /pytorch/torch/csrc/distributed/rpc/script_remote_call.cpp:33:20", " #3 0xe4b1ed5 in torch::distributed::rpc::ScriptRemoteCall::fromMessage(torch::distributed::rpc::Message const&) /pytorch/torch/csrc/distributed/rpc/script_remote_call.cpp:80:10", " #4 0xe55f8a0 in torch::distributed::rpc::deserializeRequest(torch::distributed::rpc::Message const&) /pytorch/torch/csrc/distributed/rpc/utils.cpp:108:14", " #5 0x6120a8 in LLVMFuzzerTestOneInput /message_deserialize.cc:192:27", " #6 0x535de1 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) /llvm-project-llvmorg-14.0.6/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:611:15", " #7 0x51fcec in fuzzer::RunOneTest(fuzzer::Fuzzer*, char const*, unsigned long) /llvm-project-llvmorg-14.0.6/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:324:6", " #8 0x525a3b in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /llvm-project-llvmorg-14.0.6/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:860:9", " #9 0x54eff2 in main /llvm-project-llvmorg-14.0.6/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10", " #10 0x7ffff7a37082 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x24082) (BuildId: 1878e6b475720c7c51969e69ab2d276fae6d1dee)", " #11 0x51a60d in _start (/message_deserialize_fuzz+0x51a60d)", "", "AddressSanitizer can not provide additional info.", "SUMMARY: AddressSanitizer: SEGV /pytorch/aten/src/ATen/core/ivalue.h:526:26 in c10::IValue::isTuple() const", "==8528==ABORTING" ``` Pull Request resolved: https://github.com/pytorch/pytorch/pull/102156 Approved by: https://github.com/ezyang
373 lines
10 KiB
C++
373 lines
10 KiB
C++
//===--- ArrayRef.h - Array Reference Wrapper -------------------*- C++ -*-===//
|
|
//
|
|
// The LLVM Compiler Infrastructure
|
|
//
|
|
// This file is distributed under the University of Illinois Open Source
|
|
// License. See LICENSE.TXT for details.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// ATen: modified from llvm::ArrayRef.
|
|
// removed llvm-specific functionality
|
|
// removed some implicit const -> non-const conversions that rely on
|
|
// complicated std::enable_if meta-programming
|
|
// removed a bunch of slice variants for simplicity...
|
|
|
|
#pragma once
|
|
|
|
#include <c10/util/C++17.h>
|
|
#include <c10/util/Deprecated.h>
|
|
#include <c10/util/Exception.h>
|
|
#include <c10/util/SmallVector.h>
|
|
|
|
#include <array>
|
|
#include <iterator>
|
|
#include <vector>
|
|
|
|
namespace c10 {
|
|
/// ArrayRef - Represent a constant reference to an array (0 or more elements
|
|
/// consecutively in memory), i.e. a start pointer and a length. It allows
|
|
/// various APIs to take consecutive elements easily and conveniently.
|
|
///
|
|
/// This class does not own the underlying data, it is expected to be used in
|
|
/// situations where the data resides in some other buffer, whose lifetime
|
|
/// extends past that of the ArrayRef. For this reason, it is not in general
|
|
/// safe to store an ArrayRef.
|
|
///
|
|
/// This is intended to be trivially copyable, so it should be passed by
|
|
/// value.
|
|
template <typename T>
|
|
class ArrayRef final {
|
|
public:
|
|
using iterator = const T*;
|
|
using const_iterator = const T*;
|
|
using size_type = size_t;
|
|
using value_type = T;
|
|
|
|
using reverse_iterator = std::reverse_iterator<iterator>;
|
|
|
|
private:
|
|
/// The start of the array, in an external buffer.
|
|
const T* Data;
|
|
|
|
/// The number of elements.
|
|
size_type Length;
|
|
|
|
void debugCheckNullptrInvariant() {
|
|
TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
|
|
Data != nullptr || Length == 0,
|
|
"created ArrayRef with nullptr and non-zero length! c10::optional relies on this being illegal");
|
|
}
|
|
|
|
public:
|
|
/// @name Constructors
|
|
/// @{
|
|
|
|
/// Construct an empty ArrayRef.
|
|
/* implicit */ constexpr ArrayRef() : Data(nullptr), Length(0) {}
|
|
|
|
/// Construct an ArrayRef from a single element.
|
|
// TODO Make this explicit
|
|
constexpr ArrayRef(const T& OneElt) : Data(&OneElt), Length(1) {}
|
|
|
|
/// Construct an ArrayRef from a pointer and length.
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA ArrayRef(const T* data, size_t length)
|
|
: Data(data), Length(length) {
|
|
debugCheckNullptrInvariant();
|
|
}
|
|
|
|
/// Construct an ArrayRef from a range.
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA ArrayRef(const T* begin, const T* end)
|
|
: Data(begin), Length(end - begin) {
|
|
debugCheckNullptrInvariant();
|
|
}
|
|
|
|
/// Construct an ArrayRef from a SmallVector. This is templated in order to
|
|
/// avoid instantiating SmallVectorTemplateCommon<T> whenever we
|
|
/// copy-construct an ArrayRef.
|
|
template <typename U>
|
|
/* implicit */ ArrayRef(const SmallVectorTemplateCommon<T, U>& Vec)
|
|
: Data(Vec.data()), Length(Vec.size()) {
|
|
debugCheckNullptrInvariant();
|
|
}
|
|
|
|
template <
|
|
typename Container,
|
|
typename = std::enable_if_t<std::is_same<
|
|
std::remove_const_t<decltype(std::declval<Container>().data())>,
|
|
T*>::value>>
|
|
/* implicit */ ArrayRef(const Container& container)
|
|
: Data(container.data()), Length(container.size()) {
|
|
debugCheckNullptrInvariant();
|
|
}
|
|
|
|
/// Construct an ArrayRef from a std::vector.
|
|
// The enable_if stuff here makes sure that this isn't used for
|
|
// std::vector<bool>, because ArrayRef can't work on a std::vector<bool>
|
|
// bitfield.
|
|
template <typename A>
|
|
/* implicit */ ArrayRef(const std::vector<T, A>& Vec)
|
|
: Data(Vec.data()), Length(Vec.size()) {
|
|
static_assert(
|
|
!std::is_same<T, bool>::value,
|
|
"ArrayRef<bool> cannot be constructed from a std::vector<bool> bitfield.");
|
|
}
|
|
|
|
/// Construct an ArrayRef from a std::array
|
|
template <size_t N>
|
|
/* implicit */ constexpr ArrayRef(const std::array<T, N>& Arr)
|
|
: Data(Arr.data()), Length(N) {}
|
|
|
|
/// Construct an ArrayRef from a C array.
|
|
template <size_t N>
|
|
/* implicit */ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {}
|
|
|
|
/// Construct an ArrayRef from a std::initializer_list.
|
|
/* implicit */ constexpr ArrayRef(const std::initializer_list<T>& Vec)
|
|
: Data(
|
|
std::begin(Vec) == std::end(Vec) ? static_cast<T*>(nullptr)
|
|
: std::begin(Vec)),
|
|
Length(Vec.size()) {}
|
|
|
|
/// @}
|
|
/// @name Simple Operations
|
|
/// @{
|
|
|
|
constexpr iterator begin() const {
|
|
return Data;
|
|
}
|
|
constexpr iterator end() const {
|
|
return Data + Length;
|
|
}
|
|
|
|
// These are actually the same as iterator, since ArrayRef only
|
|
// gives you const iterators.
|
|
constexpr const_iterator cbegin() const {
|
|
return Data;
|
|
}
|
|
constexpr const_iterator cend() const {
|
|
return Data + Length;
|
|
}
|
|
|
|
constexpr reverse_iterator rbegin() const {
|
|
return reverse_iterator(end());
|
|
}
|
|
constexpr reverse_iterator rend() const {
|
|
return reverse_iterator(begin());
|
|
}
|
|
|
|
/// empty - Check if the array is empty.
|
|
constexpr bool empty() const {
|
|
return Length == 0;
|
|
}
|
|
|
|
constexpr const T* data() const {
|
|
return Data;
|
|
}
|
|
|
|
/// size - Get the array size.
|
|
constexpr size_t size() const {
|
|
return Length;
|
|
}
|
|
|
|
/// front - Get the first element.
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA const T& front() const {
|
|
TORCH_CHECK(
|
|
!empty(), "ArrayRef: attempted to access front() of empty list");
|
|
return Data[0];
|
|
}
|
|
|
|
/// back - Get the last element.
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA const T& back() const {
|
|
TORCH_CHECK(!empty(), "ArrayRef: attempted to access back() of empty list");
|
|
return Data[Length - 1];
|
|
}
|
|
|
|
/// equals - Check for element-wise equality.
|
|
constexpr bool equals(ArrayRef RHS) const {
|
|
return Length == RHS.Length && std::equal(begin(), end(), RHS.begin());
|
|
}
|
|
|
|
/// slice(n, m) - Take M elements of the array starting at element N
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA ArrayRef<T> slice(size_t N, size_t M)
|
|
const {
|
|
TORCH_CHECK(
|
|
N + M <= size(),
|
|
"ArrayRef: invalid slice, N = ",
|
|
N,
|
|
"; M = ",
|
|
M,
|
|
"; size = ",
|
|
size());
|
|
return ArrayRef<T>(data() + N, M);
|
|
}
|
|
|
|
/// slice(n) - Chop off the first N elements of the array.
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA ArrayRef<T> slice(size_t N) const {
|
|
TORCH_CHECK(
|
|
N <= size(), "ArrayRef: invalid slice, N = ", N, "; size = ", size());
|
|
return slice(N, size() - N);
|
|
}
|
|
|
|
/// @}
|
|
/// @name Operator Overloads
|
|
/// @{
|
|
constexpr const T& operator[](size_t Index) const {
|
|
return Data[Index];
|
|
}
|
|
|
|
/// Vector compatibility
|
|
C10_HOST_CONSTEXPR_EXCEPT_WIN_CUDA const T& at(size_t Index) const {
|
|
TORCH_CHECK(
|
|
Index < Length,
|
|
"ArrayRef: invalid index Index = ",
|
|
Index,
|
|
"; Length = ",
|
|
Length);
|
|
return Data[Index];
|
|
}
|
|
|
|
/// Disallow accidental assignment from a temporary.
|
|
///
|
|
/// The declaration here is extra complicated so that "arrayRef = {}"
|
|
/// continues to select the move assignment operator.
|
|
template <typename U>
|
|
typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type&
|
|
operator=(U&& Temporary) = delete;
|
|
|
|
/// Disallow accidental assignment from a temporary.
|
|
///
|
|
/// The declaration here is extra complicated so that "arrayRef = {}"
|
|
/// continues to select the move assignment operator.
|
|
template <typename U>
|
|
typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type&
|
|
operator=(std::initializer_list<U>) = delete;
|
|
|
|
/// @}
|
|
/// @name Expensive Operations
|
|
/// @{
|
|
std::vector<T> vec() const {
|
|
return std::vector<T>(Data, Data + Length);
|
|
}
|
|
|
|
/// @}
|
|
};
|
|
|
|
template <typename T>
|
|
std::ostream& operator<<(std::ostream& out, ArrayRef<T> list) {
|
|
int i = 0;
|
|
out << "[";
|
|
for (const auto& e : list) {
|
|
if (i++ > 0)
|
|
out << ", ";
|
|
out << e;
|
|
}
|
|
out << "]";
|
|
return out;
|
|
}
|
|
|
|
/// @name ArrayRef Convenience constructors
|
|
/// @{
|
|
|
|
/// Construct an ArrayRef from a single element.
|
|
template <typename T>
|
|
ArrayRef<T> makeArrayRef(const T& OneElt) {
|
|
return OneElt;
|
|
}
|
|
|
|
/// Construct an ArrayRef from a pointer and length.
|
|
template <typename T>
|
|
ArrayRef<T> makeArrayRef(const T* data, size_t length) {
|
|
return ArrayRef<T>(data, length);
|
|
}
|
|
|
|
/// Construct an ArrayRef from a range.
|
|
template <typename T>
|
|
ArrayRef<T> makeArrayRef(const T* begin, const T* end) {
|
|
return ArrayRef<T>(begin, end);
|
|
}
|
|
|
|
/// Construct an ArrayRef from a SmallVector.
|
|
template <typename T>
|
|
ArrayRef<T> makeArrayRef(const SmallVectorImpl<T>& Vec) {
|
|
return Vec;
|
|
}
|
|
|
|
/// Construct an ArrayRef from a SmallVector.
|
|
template <typename T, unsigned N>
|
|
ArrayRef<T> makeArrayRef(const SmallVector<T, N>& Vec) {
|
|
return Vec;
|
|
}
|
|
|
|
/// Construct an ArrayRef from a std::vector.
|
|
template <typename T>
|
|
ArrayRef<T> makeArrayRef(const std::vector<T>& Vec) {
|
|
return Vec;
|
|
}
|
|
|
|
/// Construct an ArrayRef from a std::array.
|
|
template <typename T, std::size_t N>
|
|
ArrayRef<T> makeArrayRef(const std::array<T, N>& Arr) {
|
|
return Arr;
|
|
}
|
|
|
|
/// Construct an ArrayRef from an ArrayRef (no-op) (const)
|
|
template <typename T>
|
|
ArrayRef<T> makeArrayRef(const ArrayRef<T>& Vec) {
|
|
return Vec;
|
|
}
|
|
|
|
/// Construct an ArrayRef from an ArrayRef (no-op)
|
|
template <typename T>
|
|
ArrayRef<T>& makeArrayRef(ArrayRef<T>& Vec) {
|
|
return Vec;
|
|
}
|
|
|
|
/// Construct an ArrayRef from a C array.
|
|
template <typename T, size_t N>
|
|
ArrayRef<T> makeArrayRef(const T (&Arr)[N]) {
|
|
return ArrayRef<T>(Arr);
|
|
}
|
|
|
|
// WARNING: Template instantiation will NOT be willing to do an implicit
|
|
// conversions to get you to an c10::ArrayRef, which is why we need so
|
|
// many overloads.
|
|
|
|
template <typename T>
|
|
bool operator==(c10::ArrayRef<T> a1, c10::ArrayRef<T> a2) {
|
|
return a1.equals(a2);
|
|
}
|
|
|
|
template <typename T>
|
|
bool operator!=(c10::ArrayRef<T> a1, c10::ArrayRef<T> a2) {
|
|
return !a1.equals(a2);
|
|
}
|
|
|
|
template <typename T>
|
|
bool operator==(const std::vector<T>& a1, c10::ArrayRef<T> a2) {
|
|
return c10::ArrayRef<T>(a1).equals(a2);
|
|
}
|
|
|
|
template <typename T>
|
|
bool operator!=(const std::vector<T>& a1, c10::ArrayRef<T> a2) {
|
|
return !c10::ArrayRef<T>(a1).equals(a2);
|
|
}
|
|
|
|
template <typename T>
|
|
bool operator==(c10::ArrayRef<T> a1, const std::vector<T>& a2) {
|
|
return a1.equals(c10::ArrayRef<T>(a2));
|
|
}
|
|
|
|
template <typename T>
|
|
bool operator!=(c10::ArrayRef<T> a1, const std::vector<T>& a2) {
|
|
return !a1.equals(c10::ArrayRef<T>(a2));
|
|
}
|
|
|
|
using IntArrayRef = ArrayRef<int64_t>;
|
|
|
|
// This alias is deprecated because it doesn't make ownership
|
|
// semantics obvious. Use IntArrayRef instead!
|
|
C10_DEFINE_DEPRECATED_USING(IntList, ArrayRef<int64_t>)
|
|
|
|
} // namespace c10
|