mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 12:54:11 +08:00
Revert "Fix clang-tidy warnings in Caffe2 code (#134935)"
This reverts commit 7cfd23636c8fa6fcbb8bf3ea34e15b847ec9ad9d. Reverted https://github.com/pytorch/pytorch/pull/134935 on behalf of https://github.com/izaitsevfb due to breaks internal builds, caffe2 is still used internally ([comment](https://github.com/pytorch/pytorch/pull/134935#issuecomment-2349368152))
This commit is contained in:
@ -210,8 +210,6 @@ include_patterns = [
|
|||||||
'aten/src/ATen/native/nested/*.h',
|
'aten/src/ATen/native/nested/*.h',
|
||||||
'c10/**/*.cpp',
|
'c10/**/*.cpp',
|
||||||
'c10/**/*.h',
|
'c10/**/*.h',
|
||||||
'caffe2/**/*.cc',
|
|
||||||
'caffe2/**/*.h',
|
|
||||||
'torch/*.h',
|
'torch/*.h',
|
||||||
'torch/csrc/*.h',
|
'torch/csrc/*.h',
|
||||||
'torch/csrc/*.cpp',
|
'torch/csrc/*.cpp',
|
||||||
|
1
BUCK.oss
1
BUCK.oss
@ -65,6 +65,7 @@ cxx_library(
|
|||||||
"caffe2/serialize/file_adapter.cc",
|
"caffe2/serialize/file_adapter.cc",
|
||||||
"caffe2/serialize/inline_container.cc",
|
"caffe2/serialize/inline_container.cc",
|
||||||
"caffe2/serialize/istream_adapter.cc",
|
"caffe2/serialize/istream_adapter.cc",
|
||||||
|
"caffe2/serialize/read_adapter_interface.cc",
|
||||||
],
|
],
|
||||||
visibility = ["PUBLIC"],
|
visibility = ["PUBLIC"],
|
||||||
deps = [
|
deps = [
|
||||||
|
@ -473,6 +473,7 @@ filegroup(
|
|||||||
"caffe2/serialize/file_adapter.cc",
|
"caffe2/serialize/file_adapter.cc",
|
||||||
"caffe2/serialize/inline_container.cc",
|
"caffe2/serialize/inline_container.cc",
|
||||||
"caffe2/serialize/istream_adapter.cc",
|
"caffe2/serialize/istream_adapter.cc",
|
||||||
|
"caffe2/serialize/read_adapter_interface.cc",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -34,6 +34,7 @@ def define_targets(rules):
|
|||||||
"caffe2/serialize/file_adapter.cc",
|
"caffe2/serialize/file_adapter.cc",
|
||||||
"caffe2/serialize/inline_container.cc",
|
"caffe2/serialize/inline_container.cc",
|
||||||
"caffe2/serialize/istream_adapter.cc",
|
"caffe2/serialize/istream_adapter.cc",
|
||||||
|
"caffe2/serialize/read_adapter_interface.cc",
|
||||||
],
|
],
|
||||||
copts = ["-fexceptions"],
|
copts = ["-fexceptions"],
|
||||||
tags = [
|
tags = [
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
|
||||||
#include <c10/macros/Macros.h>
|
#include "caffe2/core/common.h"
|
||||||
|
|
||||||
namespace caffe2 {
|
namespace caffe2 {
|
||||||
|
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
// example, if your compiler did not specify -mavx, you should not provide
|
// example, if your compiler did not specify -mavx, you should not provide
|
||||||
// the CAFFE2_PERF_WITH_AVX macro.
|
// the CAFFE2_PERF_WITH_AVX macro.
|
||||||
|
|
||||||
#include "caffe2/core/macros.h"
|
#include "caffe2/core/common.h"
|
||||||
|
|
||||||
#ifdef CAFFE2_PERF_WITH_AVX
|
#ifdef CAFFE2_PERF_WITH_AVX
|
||||||
#ifndef __AVX__
|
#ifndef __AVX__
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
// example, if your compiler did not specify -mavx2, you should not provide
|
// example, if your compiler did not specify -mavx2, you should not provide
|
||||||
// the CAFFE2_PERF_WITH_AVX2 macro.
|
// the CAFFE2_PERF_WITH_AVX2 macro.
|
||||||
|
|
||||||
#include "caffe2/core/macros.h"
|
#include "caffe2/core/common.h"
|
||||||
|
|
||||||
#ifdef CAFFE2_PERF_WITH_AVX2
|
#ifdef CAFFE2_PERF_WITH_AVX2
|
||||||
#ifndef __AVX2__
|
#ifndef __AVX2__
|
||||||
|
@ -6,7 +6,8 @@ list(APPEND Caffe2_CPU_SRCS
|
|||||||
${CMAKE_CURRENT_SOURCE_DIR}/inline_container.cc
|
${CMAKE_CURRENT_SOURCE_DIR}/inline_container.cc
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/istream_adapter.cc
|
${CMAKE_CURRENT_SOURCE_DIR}/istream_adapter.cc
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/file_adapter.cc
|
${CMAKE_CURRENT_SOURCE_DIR}/file_adapter.cc
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/crc.cc)
|
${CMAKE_CURRENT_SOURCE_DIR}/crc.cc
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/read_adapter_interface.cc)
|
||||||
list(APPEND Caffe2_CPU_INCLUDE ${PROJECT_SOURCE_DIR}/third_party/miniz-2.1.0)
|
list(APPEND Caffe2_CPU_INCLUDE ${PROJECT_SOURCE_DIR}/third_party/miniz-2.1.0)
|
||||||
|
|
||||||
set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} PARENT_SCOPE)
|
set(Caffe2_CPU_TEST_SRCS ${Caffe2_CPU_TEST_SRCS} PARENT_SCOPE)
|
||||||
|
@ -25,7 +25,7 @@
|
|||||||
// using the aforementioned #defines the table is automatically fitted to your needs
|
// using the aforementioned #defines the table is automatically fitted to your needs
|
||||||
|
|
||||||
// uint8_t, uint32_t, int32_t
|
// uint8_t, uint32_t, int32_t
|
||||||
#include <cstdint>
|
#include <stdint.h>
|
||||||
// size_t
|
// size_t
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
|
|
||||||
|
@ -3,11 +3,13 @@
|
|||||||
#include <cerrno>
|
#include <cerrno>
|
||||||
#include <cstdio>
|
#include <cstdio>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include "caffe2/core/common.h"
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
FileAdapter::RAIIFile::RAIIFile(const std::string& file_name)
|
FileAdapter::RAIIFile::RAIIFile(const std::string& file_name) {
|
||||||
: fp_(fopen(file_name.c_str(), "rb")) {
|
fp_ = fopen(file_name.c_str(), "rb");
|
||||||
if (fp_ == nullptr) {
|
if (fp_ == nullptr) {
|
||||||
auto old_errno = errno;
|
auto old_errno = errno;
|
||||||
#if defined(_WIN32) && (defined(__MINGW32__) || defined(_MSC_VER))
|
#if defined(_WIN32) && (defined(__MINGW32__) || defined(_MSC_VER))
|
||||||
@ -75,4 +77,5 @@ size_t FileAdapter::read(uint64_t pos, void* buf, size_t n, const char* what)
|
|||||||
|
|
||||||
FileAdapter::~FileAdapter() = default;
|
FileAdapter::~FileAdapter() = default;
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -1,11 +1,14 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <memory>
|
||||||
#include <c10/macros/Macros.h>
|
#include <c10/macros/Macros.h>
|
||||||
#include <string>
|
|
||||||
|
|
||||||
|
#include "caffe2/serialize/istream_adapter.h"
|
||||||
#include "caffe2/serialize/read_adapter_interface.h"
|
#include "caffe2/serialize/read_adapter_interface.h"
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
class TORCH_API FileAdapter final : public ReadAdapterInterface {
|
class TORCH_API FileAdapter final : public ReadAdapterInterface {
|
||||||
public:
|
public:
|
||||||
@ -29,4 +32,5 @@ class TORCH_API FileAdapter final : public ReadAdapterInterface {
|
|||||||
uint64_t size_;
|
uint64_t size_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -1,9 +1,10 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
#include <caffe2/serialize/read_adapter_interface.h>
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
|
#include <caffe2/serialize/read_adapter_interface.h>
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
|
||||||
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
class MemoryReadAdapter final : public caffe2::serialize::ReadAdapterInterface {
|
class MemoryReadAdapter final : public caffe2::serialize::ReadAdapterInterface {
|
||||||
public:
|
public:
|
||||||
@ -14,18 +15,18 @@ class MemoryReadAdapter final : public caffe2::serialize::ReadAdapterInterface {
|
|||||||
return size_;
|
return size_;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t read(
|
size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
|
||||||
uint64_t pos,
|
const override {
|
||||||
void* buf,
|
(void) what;
|
||||||
size_t n,
|
|
||||||
const char* what [[maybe_unused]] = "") const override {
|
|
||||||
memcpy(buf, (int8_t*)(data_) + pos, n);
|
memcpy(buf, (int8_t*)(data_) + pos, n);
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const void* data_;
|
const void* data_;
|
||||||
off_t size_{};
|
off_t size_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
|
||||||
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -10,11 +10,15 @@
|
|||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
|
#include <c10/core/Allocator.h>
|
||||||
|
#include <c10/core/Backend.h>
|
||||||
#include <c10/core/CPUAllocator.h>
|
#include <c10/core/CPUAllocator.h>
|
||||||
|
#include <c10/core/Backend.h>
|
||||||
#include <c10/util/Exception.h>
|
#include <c10/util/Exception.h>
|
||||||
#include <c10/util/Logging.h>
|
#include <c10/util/Logging.h>
|
||||||
#include <c10/util/hash.h>
|
#include <c10/util/hash.h>
|
||||||
|
|
||||||
|
#include "caffe2/core/common.h"
|
||||||
#include "caffe2/serialize/file_adapter.h"
|
#include "caffe2/serialize/file_adapter.h"
|
||||||
#include "caffe2/serialize/inline_container.h"
|
#include "caffe2/serialize/inline_container.h"
|
||||||
#include "caffe2/serialize/istream_adapter.h"
|
#include "caffe2/serialize/istream_adapter.h"
|
||||||
@ -23,8 +27,8 @@
|
|||||||
#include "caffe2/serialize/versions.h"
|
#include "caffe2/serialize/versions.h"
|
||||||
#include "miniz.h"
|
#include "miniz.h"
|
||||||
|
|
||||||
|
namespace caffe2 {
|
||||||
namespace caffe2::serialize {
|
namespace serialize {
|
||||||
constexpr c10::string_view kDebugPklSuffix(".debug_pkl");
|
constexpr c10::string_view kDebugPklSuffix(".debug_pkl");
|
||||||
|
|
||||||
struct MzZipReaderIterWrapper {
|
struct MzZipReaderIterWrapper {
|
||||||
@ -190,7 +194,8 @@ void PyTorchStreamReader::init() {
|
|||||||
|
|
||||||
// version check
|
// version check
|
||||||
at::DataPtr version_ptr;
|
at::DataPtr version_ptr;
|
||||||
size_t version_size = 0;
|
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
|
||||||
|
size_t version_size;
|
||||||
if (hasRecord(".data/version")) {
|
if (hasRecord(".data/version")) {
|
||||||
std::tie(version_ptr, version_size) = getRecord(".data/version");
|
std::tie(version_ptr, version_size) = getRecord(".data/version");
|
||||||
} else {
|
} else {
|
||||||
@ -199,7 +204,7 @@ void PyTorchStreamReader::init() {
|
|||||||
}
|
}
|
||||||
std::string version(static_cast<const char*>(version_ptr.get()), version_size);
|
std::string version(static_cast<const char*>(version_ptr.get()), version_size);
|
||||||
try {
|
try {
|
||||||
version_ = std::stoll(version);
|
version_ = std::stoull(version);
|
||||||
} catch (const std::invalid_argument& e) {
|
} catch (const std::invalid_argument& e) {
|
||||||
CAFFE_THROW("Couldn't parse the version ",
|
CAFFE_THROW("Couldn't parse the version ",
|
||||||
version,
|
version,
|
||||||
@ -622,7 +627,7 @@ PyTorchStreamWriter::PyTorchStreamWriter(const std::string& file_name)
|
|||||||
}
|
}
|
||||||
|
|
||||||
PyTorchStreamWriter::PyTorchStreamWriter(
|
PyTorchStreamWriter::PyTorchStreamWriter(
|
||||||
const std::function<size_t(const void*, size_t)>& writer_func)
|
const std::function<size_t(const void*, size_t)> writer_func)
|
||||||
: archive_name_("archive"),
|
: archive_name_("archive"),
|
||||||
writer_func_(writer_func) {
|
writer_func_(writer_func) {
|
||||||
setup(archive_name_);
|
setup(archive_name_);
|
||||||
@ -633,7 +638,7 @@ void PyTorchStreamWriter::setup(const string& file_name) {
|
|||||||
memset(ar_.get(), 0, sizeof(mz_zip_archive));
|
memset(ar_.get(), 0, sizeof(mz_zip_archive));
|
||||||
archive_name_plus_slash_ = archive_name_ + "/"; // for writeRecord().
|
archive_name_plus_slash_ = archive_name_ + "/"; // for writeRecord().
|
||||||
|
|
||||||
if (archive_name_.empty()) {
|
if (archive_name_.size() == 0) {
|
||||||
CAFFE_THROW("invalid file name: ", file_name);
|
CAFFE_THROW("invalid file name: ", file_name);
|
||||||
}
|
}
|
||||||
if (!writer_func_) {
|
if (!writer_func_) {
|
||||||
@ -644,7 +649,7 @@ void PyTorchStreamWriter::setup(const string& file_name) {
|
|||||||
|
|
||||||
const std::string dir_name = parentdir(file_name);
|
const std::string dir_name = parentdir(file_name);
|
||||||
if(!dir_name.empty()) {
|
if(!dir_name.empty()) {
|
||||||
struct stat st{};
|
struct stat st;
|
||||||
bool dir_exists = (stat(dir_name.c_str(), &st) == 0 && (st.st_mode & S_IFDIR));
|
bool dir_exists = (stat(dir_name.c_str(), &st) == 0 && (st.st_mode & S_IFDIR));
|
||||||
TORCH_CHECK(dir_exists, "Parent directory ", dir_name, " does not exist.");
|
TORCH_CHECK(dir_exists, "Parent directory ", dir_name, " does not exist.");
|
||||||
}
|
}
|
||||||
@ -701,8 +706,8 @@ void PyTorchStreamWriter::writeRecord(
|
|||||||
/*uncomp_size=*/0,
|
/*uncomp_size=*/0,
|
||||||
/*uncomp_crc32=*/0,
|
/*uncomp_crc32=*/0,
|
||||||
/*last_modified=*/nullptr,
|
/*last_modified=*/nullptr,
|
||||||
/*user_extra_data_local=*/padding_.c_str(),
|
/*user_extra_data=*/padding_.c_str(),
|
||||||
/*user_extra_data_local_len=*/padding_size,
|
/*user_extra_data_len=*/padding_size,
|
||||||
/*user_extra_data_central=*/nullptr,
|
/*user_extra_data_central=*/nullptr,
|
||||||
/*user_extra_data_central_len=*/0);
|
/*user_extra_data_central_len=*/0);
|
||||||
valid("writing file ", name.c_str());
|
valid("writing file ", name.c_str());
|
||||||
@ -815,4 +820,5 @@ PyTorchStreamWriter::~PyTorchStreamWriter() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -6,6 +6,7 @@
|
|||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <istream>
|
#include <istream>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
#include <ostream>
|
||||||
#include <unordered_set>
|
#include <unordered_set>
|
||||||
|
|
||||||
#include <c10/core/Allocator.h>
|
#include <c10/core/Allocator.h>
|
||||||
@ -90,8 +91,8 @@ typedef struct mz_zip_archive mz_zip_archive;
|
|||||||
// model.json as the last file when writing after we have accumulated all
|
// model.json as the last file when writing after we have accumulated all
|
||||||
// other information.
|
// other information.
|
||||||
|
|
||||||
|
namespace caffe2 {
|
||||||
namespace caffe2::serialize {
|
namespace serialize {
|
||||||
|
|
||||||
static constexpr const char* kSerializationIdRecordName = ".data/serialization_id";
|
static constexpr const char* kSerializationIdRecordName = ".data/serialization_id";
|
||||||
|
|
||||||
@ -195,18 +196,18 @@ class TORCH_API PyTorchStreamReader final {
|
|||||||
std::string archive_name_;
|
std::string archive_name_;
|
||||||
std::string archive_name_plus_slash_;
|
std::string archive_name_plus_slash_;
|
||||||
std::shared_ptr<ReadAdapterInterface> in_;
|
std::shared_ptr<ReadAdapterInterface> in_;
|
||||||
int64_t version_{};
|
int64_t version_;
|
||||||
std::mutex reader_lock_;
|
std::mutex reader_lock_;
|
||||||
bool load_debug_symbol_ = true;
|
bool load_debug_symbol_ = true;
|
||||||
std::string serialization_id_;
|
std::string serialization_id_;
|
||||||
size_t additional_reader_size_threshold_{};
|
size_t additional_reader_size_threshold_;
|
||||||
};
|
};
|
||||||
|
|
||||||
class TORCH_API PyTorchStreamWriter final {
|
class TORCH_API PyTorchStreamWriter final {
|
||||||
public:
|
public:
|
||||||
explicit PyTorchStreamWriter(const std::string& archive_name);
|
explicit PyTorchStreamWriter(const std::string& archive_name);
|
||||||
explicit PyTorchStreamWriter(
|
explicit PyTorchStreamWriter(
|
||||||
const std::function<size_t(const void*, size_t)>& writer_func);
|
const std::function<size_t(const void*, size_t)> writer_func);
|
||||||
|
|
||||||
void setMinVersion(const uint64_t version);
|
void setMinVersion(const uint64_t version);
|
||||||
|
|
||||||
@ -273,4 +274,5 @@ size_t getPadding(
|
|||||||
std::string& padding_buf);
|
std::string& padding_buf);
|
||||||
} // namespace detail
|
} // namespace detail
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -5,14 +5,12 @@
|
|||||||
|
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
#include <c10/util/Logging.h>
|
|
||||||
#include <c10/util/irange.h>
|
|
||||||
#include "caffe2/serialize/inline_container.h"
|
#include "caffe2/serialize/inline_container.h"
|
||||||
#include "caffe2/serialize/istream_adapter.h"
|
#include <c10/util/Logging.h>
|
||||||
|
#include "c10/util/irange.h"
|
||||||
|
|
||||||
|
namespace caffe2 {
|
||||||
// NOLINTBEGIN(*-narrowing-conversions)
|
namespace serialize {
|
||||||
namespace caffe2::serialize {
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
TEST(PyTorchStreamWriterAndReader, SaveAndLoad) {
|
TEST(PyTorchStreamWriterAndReader, SaveAndLoad) {
|
||||||
@ -21,7 +19,7 @@ TEST(PyTorchStreamWriterAndReader, SaveAndLoad) {
|
|||||||
std::ostringstream oss;
|
std::ostringstream oss;
|
||||||
// write records through writers
|
// write records through writers
|
||||||
PyTorchStreamWriter writer([&](const void* b, size_t n) -> size_t {
|
PyTorchStreamWriter writer([&](const void* b, size_t n) -> size_t {
|
||||||
oss.write(static_cast<const char*>(b), static_cast<std::streamsize>(n));
|
oss.write(static_cast<const char*>(b), n);
|
||||||
return oss ? n : 0;
|
return oss ? n : 0;
|
||||||
});
|
});
|
||||||
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,cppcoreguidelines-avoid-magic-numbers)
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,cppcoreguidelines-avoid-magic-numbers)
|
||||||
@ -30,14 +28,14 @@ TEST(PyTorchStreamWriterAndReader, SaveAndLoad) {
|
|||||||
std::vector<uint8_t> buf(data1.size());
|
std::vector<uint8_t> buf(data1.size());
|
||||||
|
|
||||||
for (auto i : c10::irange(data1.size())) {
|
for (auto i : c10::irange(data1.size())) {
|
||||||
data1[i] = static_cast<char>(data1.size() - i);
|
data1[i] = data1.size() - i;
|
||||||
}
|
}
|
||||||
writer.writeRecord("key1", data1.data(), data1.size());
|
writer.writeRecord("key1", data1.data(), data1.size());
|
||||||
|
|
||||||
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,cppcoreguidelines-avoid-magic-numbers)
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,cppcoreguidelines-avoid-magic-numbers)
|
||||||
std::array<char, 64> data2;
|
std::array<char, 64> data2;
|
||||||
for (auto i : c10::irange(data2.size())) {
|
for (auto i : c10::irange(data2.size())) {
|
||||||
data2[i] = static_cast<char>(data2.size() - i);
|
data2[i] = data2.size() - i;
|
||||||
}
|
}
|
||||||
writer.writeRecord("key2", data2.data(), data2.size());
|
writer.writeRecord("key2", data2.data(), data2.size());
|
||||||
|
|
||||||
@ -151,7 +149,7 @@ TEST(PyTorchStreamWriterAndReader, LoadWithMultiThreads) {
|
|||||||
PyTorchStreamReader reader(&iss);
|
PyTorchStreamReader reader(&iss);
|
||||||
reader.setAdditionalReaderSizeThreshold(0);
|
reader.setAdditionalReaderSizeThreshold(0);
|
||||||
// before testing, sanity check
|
// before testing, sanity check
|
||||||
int64_t size1 = 0, size2 = 0, ret = 0;
|
int64_t size1, size2, ret;
|
||||||
at::DataPtr data_ptr;
|
at::DataPtr data_ptr;
|
||||||
std::tie(data_ptr, size1) = reader.getRecord("key1");
|
std::tie(data_ptr, size1) = reader.getRecord("key1");
|
||||||
std::tie(data_ptr, size2) = reader.getRecord("key2");
|
std::tie(data_ptr, size2) = reader.getRecord("key2");
|
||||||
@ -298,7 +296,7 @@ TEST(PytorchStreamWriterAndReader, SkipDebugRecords) {
|
|||||||
reader.setShouldLoadDebugSymbol(false);
|
reader.setShouldLoadDebugSymbol(false);
|
||||||
EXPECT_FALSE(reader.hasRecord("key1.debug_pkl"));
|
EXPECT_FALSE(reader.hasRecord("key1.debug_pkl"));
|
||||||
at::DataPtr ptr;
|
at::DataPtr ptr;
|
||||||
size_t size = 0;
|
size_t size;
|
||||||
std::tie(ptr, size) = reader.getRecord("key1.debug_pkl");
|
std::tie(ptr, size) = reader.getRecord("key1.debug_pkl");
|
||||||
EXPECT_EQ(size, 0);
|
EXPECT_EQ(size, 0);
|
||||||
std::vector<uint8_t> dst(data1.size());
|
std::vector<uint8_t> dst(data1.size());
|
||||||
@ -481,5 +479,5 @@ TEST_P(ChunkRecordIteratorTest, ChunkRead) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
// NOLINTEND(*-narrowing-conversions)
|
} // namespace caffe2
|
||||||
|
@ -1,7 +1,8 @@
|
|||||||
#include "caffe2/serialize/istream_adapter.h"
|
#include "caffe2/serialize/istream_adapter.h"
|
||||||
#include <c10/util/Exception.h>
|
#include <c10/util/Exception.h>
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
IStreamAdapter::IStreamAdapter(std::istream* istream) : istream_(istream) {}
|
IStreamAdapter::IStreamAdapter(std::istream* istream) : istream_(istream) {}
|
||||||
|
|
||||||
@ -32,6 +33,8 @@ void IStreamAdapter::validate(const char* what) const {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
IStreamAdapter::~IStreamAdapter() = default;
|
// NOLINTNEXTLINE(modernize-use-equals-default)
|
||||||
|
IStreamAdapter::~IStreamAdapter() {}
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -5,7 +5,8 @@
|
|||||||
#include "c10/macros/Macros.h"
|
#include "c10/macros/Macros.h"
|
||||||
#include "caffe2/serialize/read_adapter_interface.h"
|
#include "caffe2/serialize/read_adapter_interface.h"
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
// this is a reader implemented by std::istream
|
// this is a reader implemented by std::istream
|
||||||
class TORCH_API IStreamAdapter final : public ReadAdapterInterface {
|
class TORCH_API IStreamAdapter final : public ReadAdapterInterface {
|
||||||
@ -22,4 +23,5 @@ class TORCH_API IStreamAdapter final : public ReadAdapterInterface {
|
|||||||
void validate(const char* what) const;
|
void validate(const char* what) const;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
10
caffe2/serialize/read_adapter_interface.cc
Normal file
10
caffe2/serialize/read_adapter_interface.cc
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
#include "caffe2/serialize/read_adapter_interface.h"
|
||||||
|
|
||||||
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
|
// NOLINTNEXTLINE(modernize-use-equals-default)
|
||||||
|
ReadAdapterInterface::~ReadAdapterInterface() {}
|
||||||
|
|
||||||
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
@ -3,19 +3,21 @@
|
|||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
|
|
||||||
#include <c10/macros/Export.h>
|
#include "c10/macros/Macros.h"
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
// this is the interface for the (file/stream/memory) reader in
|
// this is the interface for the (file/stream/memory) reader in
|
||||||
// PyTorchStreamReader. With this interface, we can extend the support
|
// PyTorchStreamReader. with this interface, we can extend the support
|
||||||
// besides standard istream
|
// besides standard istream
|
||||||
class TORCH_API ReadAdapterInterface {
|
class TORCH_API ReadAdapterInterface {
|
||||||
public:
|
public:
|
||||||
virtual size_t size() const = 0;
|
virtual size_t size() const = 0;
|
||||||
virtual size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
|
virtual size_t read(uint64_t pos, void* buf, size_t n, const char* what = "")
|
||||||
const = 0;
|
const = 0;
|
||||||
virtual ~ReadAdapterInterface() = default;
|
virtual ~ReadAdapterInterface();
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -1,7 +1,8 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
|
|
||||||
namespace caffe2::serialize {
|
namespace caffe2 {
|
||||||
|
namespace serialize {
|
||||||
|
|
||||||
constexpr uint64_t kMinSupportedFileFormatVersion = 0x1L;
|
constexpr uint64_t kMinSupportedFileFormatVersion = 0x1L;
|
||||||
|
|
||||||
@ -128,4 +129,5 @@ constexpr uint64_t kProducedBytecodeVersion = 0x8L;
|
|||||||
constexpr uint64_t kMinSupportedBytecodeVersion = 0x4L;
|
constexpr uint64_t kMinSupportedBytecodeVersion = 0x4L;
|
||||||
constexpr uint64_t kMaxSupportedBytecodeVersion = 0x9L;
|
constexpr uint64_t kMaxSupportedBytecodeVersion = 0x9L;
|
||||||
|
|
||||||
} // namespace caffe2::serialize
|
} // namespace serialize
|
||||||
|
} // namespace caffe2
|
||||||
|
@ -51,7 +51,7 @@ size_t editDistance(
|
|||||||
(c)=(uint8_t)(s)[(i)++]; \
|
(c)=(uint8_t)(s)[(i)++]; \
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t editDistanceHelper(const char* s1,
|
int32_t editDistanceHelper(const char* s1,
|
||||||
size_t s1_len,
|
size_t s1_len,
|
||||||
const char* s2,
|
const char* s2,
|
||||||
size_t s2_len,
|
size_t s2_len,
|
||||||
|
@ -39,7 +39,7 @@ TORCH_API inline bool EndsWith(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TORCH_API size_t editDistanceHelper(
|
TORCH_API int32_t editDistanceHelper(
|
||||||
const char* s1,
|
const char* s1,
|
||||||
size_t s1_len,
|
size_t s1_len,
|
||||||
const char* s2,
|
const char* s2,
|
||||||
|
@ -1,11 +1,15 @@
|
|||||||
#ifndef CAFFE2_UTILS_THREADPOOL_H_
|
#ifndef CAFFE2_UTILS_THREADPOOL_H_
|
||||||
#define CAFFE2_UTILS_THREADPOOL_H_
|
#define CAFFE2_UTILS_THREADPOOL_H_
|
||||||
|
|
||||||
|
#include "ThreadPoolCommon.h"
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
#include <c10/macros/Macros.h>
|
#include "caffe2/core/common.h"
|
||||||
|
|
||||||
//
|
//
|
||||||
// A work-stealing threadpool loosely based off of pthreadpool
|
// A work-stealing threadpool loosely based off of pthreadpool
|
||||||
|
@ -1,9 +1,9 @@
|
|||||||
/* Standard C headers */
|
/* Standard C headers */
|
||||||
#include <cstdint>
|
#include <stdint.h>
|
||||||
|
#include <stdbool.h>
|
||||||
#include <cstdlib>
|
#include <stdlib.h>
|
||||||
#include <cstring>
|
#include <string.h>
|
||||||
#include <cassert>
|
#include <assert.h>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
|
|
||||||
#ifdef _MSC_VER
|
#ifdef _MSC_VER
|
||||||
@ -71,8 +71,8 @@ void legacy_pthreadpool_compute_1d_tiled(
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct compute_2d_context {
|
struct compute_2d_context {
|
||||||
legacy_pthreadpool_function_2d_t function{};
|
legacy_pthreadpool_function_2d_t function;
|
||||||
void* argument{};
|
void* argument;
|
||||||
caffe2::FixedDivisor<int32_t> range_j;
|
caffe2::FixedDivisor<int32_t> range_j;
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -80,8 +80,8 @@ static void compute_2d(void* context_, size_t linear_index) {
|
|||||||
TORCH_DCHECK_LE(linear_index, std::numeric_limits<int32_t>::max());
|
TORCH_DCHECK_LE(linear_index, std::numeric_limits<int32_t>::max());
|
||||||
|
|
||||||
const struct compute_2d_context* context = static_cast<compute_2d_context*>(context_);
|
const struct compute_2d_context* context = static_cast<compute_2d_context*>(context_);
|
||||||
int32_t q = 0;
|
int32_t q;
|
||||||
int32_t r = 0;
|
int32_t r;
|
||||||
context->range_j.DivMod(static_cast<int32_t>(linear_index), &q, &r);
|
context->range_j.DivMod(static_cast<int32_t>(linear_index), &q, &r);
|
||||||
context->function(context->argument, q, r);
|
context->function(context->argument, q, r);
|
||||||
}
|
}
|
||||||
@ -112,18 +112,18 @@ void legacy_pthreadpool_compute_2d(
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct compute_2d_tiled_context {
|
struct compute_2d_tiled_context {
|
||||||
legacy_pthreadpool_function_2d_tiled_t function{};
|
legacy_pthreadpool_function_2d_tiled_t function;
|
||||||
void* argument{};
|
void* argument;
|
||||||
caffe2::FixedDivisor<int32_t> tile_range_j;
|
caffe2::FixedDivisor<int32_t> tile_range_j;
|
||||||
size_t range_i{};
|
size_t range_i;
|
||||||
size_t range_j{};
|
size_t range_j;
|
||||||
size_t tile_i{};
|
size_t tile_i;
|
||||||
size_t tile_j{};
|
size_t tile_j;
|
||||||
};
|
};
|
||||||
|
|
||||||
static void compute_2d_tiled(void* context_, size_t linear_index) {
|
static void compute_2d_tiled(void* context_, size_t linear_index) {
|
||||||
int32_t q = 0;
|
int32_t q;
|
||||||
int32_t r = 0;
|
int32_t r;
|
||||||
|
|
||||||
const struct compute_2d_tiled_context* context = static_cast<compute_2d_tiled_context*>(context_);
|
const struct compute_2d_tiled_context* context = static_cast<compute_2d_tiled_context*>(context_);
|
||||||
context->tile_range_j.DivMod(linear_index, &q, &r);
|
context->tile_range_j.DivMod(linear_index, &q, &r);
|
||||||
@ -172,26 +172,26 @@ void legacy_pthreadpool_compute_2d_tiled(
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct compute_3d_tiled_context {
|
struct compute_3d_tiled_context {
|
||||||
legacy_pthreadpool_function_3d_tiled_t function{};
|
legacy_pthreadpool_function_3d_tiled_t function;
|
||||||
void* argument{};
|
void* argument;
|
||||||
caffe2::FixedDivisor<int32_t> tile_range_j;
|
caffe2::FixedDivisor<int32_t> tile_range_j;
|
||||||
caffe2::FixedDivisor<int32_t> tile_range_k;
|
caffe2::FixedDivisor<int32_t> tile_range_k;
|
||||||
size_t range_i{};
|
size_t range_i;
|
||||||
size_t range_j{};
|
size_t range_j;
|
||||||
size_t range_k{};
|
size_t range_k;
|
||||||
size_t tile_i{};
|
size_t tile_i;
|
||||||
size_t tile_j{};
|
size_t tile_j;
|
||||||
size_t tile_k{};
|
size_t tile_k;
|
||||||
};
|
};
|
||||||
|
|
||||||
static void compute_3d_tiled(
|
static void compute_3d_tiled(
|
||||||
void* context_,
|
void* context_,
|
||||||
size_t linear_index) {
|
size_t linear_index) {
|
||||||
int32_t tile_index_ij = 0, tile_index_k = 0;
|
int32_t tile_index_ij, tile_index_k;
|
||||||
const struct compute_3d_tiled_context* context = static_cast<compute_3d_tiled_context*>(context_);
|
const struct compute_3d_tiled_context* context = static_cast<compute_3d_tiled_context*>(context_);
|
||||||
context->tile_range_k.DivMod(
|
context->tile_range_k.DivMod(
|
||||||
static_cast<int32_t>(linear_index), &tile_index_ij, &tile_index_k);
|
static_cast<int32_t>(linear_index), &tile_index_ij, &tile_index_k);
|
||||||
int32_t tile_index_i = 0, tile_index_j = 0;
|
int32_t tile_index_i, tile_index_j;
|
||||||
context->tile_range_j.DivMod(tile_index_ij, &tile_index_i, &tile_index_j);
|
context->tile_range_j.DivMod(tile_index_ij, &tile_index_i, &tile_index_j);
|
||||||
const size_t max_tile_i = context->tile_i;
|
const size_t max_tile_i = context->tile_i;
|
||||||
const size_t max_tile_j = context->tile_j;
|
const size_t max_tile_j = context->tile_j;
|
||||||
@ -261,31 +261,31 @@ void legacy_pthreadpool_compute_3d_tiled(
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct compute_4d_tiled_context {
|
struct compute_4d_tiled_context {
|
||||||
legacy_pthreadpool_function_4d_tiled_t function{};
|
legacy_pthreadpool_function_4d_tiled_t function;
|
||||||
void* argument{};
|
void* argument;
|
||||||
caffe2::FixedDivisor<int32_t> tile_range_kl;
|
caffe2::FixedDivisor<int32_t> tile_range_kl;
|
||||||
caffe2::FixedDivisor<int32_t> tile_range_j;
|
caffe2::FixedDivisor<int32_t> tile_range_j;
|
||||||
caffe2::FixedDivisor<int32_t> tile_range_l;
|
caffe2::FixedDivisor<int32_t> tile_range_l;
|
||||||
size_t range_i{};
|
size_t range_i;
|
||||||
size_t range_j{};
|
size_t range_j;
|
||||||
size_t range_k{};
|
size_t range_k;
|
||||||
size_t range_l{};
|
size_t range_l;
|
||||||
size_t tile_i{};
|
size_t tile_i;
|
||||||
size_t tile_j{};
|
size_t tile_j;
|
||||||
size_t tile_k{};
|
size_t tile_k;
|
||||||
size_t tile_l{};
|
size_t tile_l;
|
||||||
};
|
};
|
||||||
|
|
||||||
static void compute_4d_tiled(
|
static void compute_4d_tiled(
|
||||||
void* context_,
|
void* context_,
|
||||||
size_t linear_index) {
|
size_t linear_index) {
|
||||||
int32_t tile_index_ij = 0, tile_index_kl = 0;
|
int32_t tile_index_ij, tile_index_kl;
|
||||||
const struct compute_4d_tiled_context* context = static_cast<compute_4d_tiled_context*>(context_);
|
const struct compute_4d_tiled_context* context = static_cast<compute_4d_tiled_context*>(context_);
|
||||||
context->tile_range_kl.DivMod(
|
context->tile_range_kl.DivMod(
|
||||||
static_cast<int32_t>(linear_index), &tile_index_ij, &tile_index_kl);
|
static_cast<int32_t>(linear_index), &tile_index_ij, &tile_index_kl);
|
||||||
int32_t tile_index_i = 0, tile_index_j = 0;
|
int32_t tile_index_i, tile_index_j;
|
||||||
context->tile_range_j.DivMod(tile_index_ij, &tile_index_i, &tile_index_j);
|
context->tile_range_j.DivMod(tile_index_ij, &tile_index_i, &tile_index_j);
|
||||||
int32_t tile_index_k = 0, tile_index_l = 0;
|
int32_t tile_index_k, tile_index_l;
|
||||||
context->tile_range_l.DivMod(tile_index_kl, &tile_index_k, &tile_index_l);
|
context->tile_range_l.DivMod(tile_index_kl, &tile_index_k, &tile_index_l);
|
||||||
const size_t max_tile_i = context->tile_i;
|
const size_t max_tile_i = context->tile_i;
|
||||||
const size_t max_tile_j = context->tile_j;
|
const size_t max_tile_j = context->tile_j;
|
||||||
|
@ -7,7 +7,9 @@ namespace caffe2 {
|
|||||||
namespace {
|
namespace {
|
||||||
static thread_local bool using_new_threadpool{false};
|
static thread_local bool using_new_threadpool{false};
|
||||||
}
|
}
|
||||||
WithCastToNewThreadPool::WithCastToNewThreadPool(bool use_new_threadpool) : use_new_threadpool_(using_new_threadpool) {
|
WithCastToNewThreadPool::WithCastToNewThreadPool(bool use_new_threadpool) {
|
||||||
|
use_new_threadpool_ = using_new_threadpool;
|
||||||
|
using_new_threadpool = use_new_threadpool;
|
||||||
}
|
}
|
||||||
WithCastToNewThreadPool::~WithCastToNewThreadPool() {
|
WithCastToNewThreadPool::~WithCastToNewThreadPool() {
|
||||||
using_new_threadpool = use_new_threadpool_;
|
using_new_threadpool = use_new_threadpool_;
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
#include <caffe2/serialize/istream_adapter.h>
|
|
||||||
#include <test/cpp/jit/test_custom_class_registrations.h>
|
#include <test/cpp/jit/test_custom_class_registrations.h>
|
||||||
#include <torch/csrc/jit/passes/freeze_module.h>
|
#include <torch/csrc/jit/passes/freeze_module.h>
|
||||||
#include <torch/custom_class.h>
|
#include <torch/custom_class.h>
|
||||||
|
@ -2,7 +2,6 @@
|
|||||||
#include <c10/util/Exception.h>
|
#include <c10/util/Exception.h>
|
||||||
#include <caffe2/serialize/file_adapter.h>
|
#include <caffe2/serialize/file_adapter.h>
|
||||||
#include <caffe2/serialize/inline_container.h>
|
#include <caffe2/serialize/inline_container.h>
|
||||||
#include <caffe2/serialize/istream_adapter.h>
|
|
||||||
#include <torch/csrc/jit/mobile/compatibility/backport_manager.h>
|
#include <torch/csrc/jit/mobile/compatibility/backport_manager.h>
|
||||||
#include <torch/csrc/jit/mobile/compatibility/model_compatibility.h>
|
#include <torch/csrc/jit/mobile/compatibility/model_compatibility.h>
|
||||||
#include <torch/csrc/jit/mobile/import.h>
|
#include <torch/csrc/jit/mobile/import.h>
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
#include <ATen/core/ivalue.h>
|
#include <ATen/core/ivalue.h>
|
||||||
#include <caffe2/serialize/file_adapter.h>
|
#include <caffe2/serialize/file_adapter.h>
|
||||||
#include <caffe2/serialize/inline_container.h>
|
#include <caffe2/serialize/inline_container.h>
|
||||||
#include <caffe2/serialize/istream_adapter.h>
|
|
||||||
#include <torch/csrc/jit/api/compilation_unit.h> // removed after using simple type_resolver/obj_loader
|
#include <torch/csrc/jit/api/compilation_unit.h> // removed after using simple type_resolver/obj_loader
|
||||||
#include <torch/csrc/jit/mobile/compatibility/model_compatibility.h>
|
#include <torch/csrc/jit/mobile/compatibility/model_compatibility.h>
|
||||||
#include <torch/csrc/jit/mobile/file_format.h>
|
#include <torch/csrc/jit/mobile/file_format.h>
|
||||||
|
Reference in New Issue
Block a user