frozenleaves/pytorch
1
0
Fork 0
mirror of https://github.com/pytorch/pytorch.git synced 2025-10-20 21:14:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[1/2] Remove caffe2 db and distributed from build system (#125092)

Browse Source 
This PR tries to decompose https://github.com/pytorch/pytorch/pull/122527 into a smaller one. Caffe2 db, distributed and some binaries have been removed.
To be noted, this was inspired and is co-dev with @r-barnes.

Pull Request resolved: https://github.com/pytorch/pytorch/pull/125092
Approved by: https://github.com/malfet
This commit is contained in:
cyy
2024-05-04 06:48:46 +00:00
committed by PyTorch MergeBot
parent 2b41e1d6fc
commit 83845a7c78
43 changed files with 0 additions and 4268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/docker/common/install_base.sh
View File

@ -113,7 +113,6 @@ install_centos() {
glibc-devel \
glibc-headers \
glog-devel \
hiredis-devel \
libstdc++-devel \
libsndfile-devel \
make \

11
.ci/docker/common/install_db.sh
View File

@ -4,11 +4,6 @@ set -ex
install_ubuntu() {
apt-get update
apt-get install -y --no-install-recommends \
libhiredis-dev \
libleveldb-dev \
liblmdb-dev \
libsnappy-dev
# Cleanup
apt-get autoclean && apt-get clean
@ -20,12 +15,6 @@ install_centos() {
# See http://fedoraproject.org/wiki/EPEL
yum --enablerepo=extras install -y epel-release
yum install -y \
hiredis-devel \
leveldb-devel \
lmdb-devel \
snappy-devel
# Cleanup
yum clean all
rm -rf /var/cache/yum

17
BUILD.bazel
View File

@ -503,16 +503,6 @@ filegroup(
],
)
filegroup(
name = "caffe2_distributed_srcs",
srcs = [
"caffe2/distributed/file_store_handler.cc",
"caffe2/distributed/file_store_handler_op.cc",
"caffe2/distributed/store_handler.cc",
"caffe2/distributed/store_ops.cc",
],
)
filegroup(
name = "caffe2_ideep_srcs",
srcs = [
@ -1011,8 +1001,6 @@ filegroup(
"caffe2/core/common_cudnn.cc",
"caffe2/core/common_gpu.cc",
"caffe2/core/event_gpu.cc",
"caffe2/db/create_db_op_gpu.cc",
"caffe2/distributed/file_store_handler_op_gpu.cc",
"caffe2/operators/communicator_op_gpu.cc",
"caffe2/operators/concat_split_op_gpu.cc",
"caffe2/operators/conv_op_cache_cudnn.cc",
@ -1262,8 +1250,6 @@ cc_library(
"caffe2/core/nomnigraph/include/nomnigraph/Support/*.h",
"caffe2/core/nomnigraph/include/nomnigraph/Transformations/*.h",
"caffe2/core/nomnigraph/tests/*.h",
"caffe2/db/*.h",
"caffe2/distributed/*.h",
"caffe2/ideep/*.h",
"caffe2/ideep/operators/*.h",
"caffe2/ideep/operators/quantization/*.h",
@ -1335,11 +1321,8 @@ cc_library(
cc_library(
name = "caffe2",
srcs = [
"caffe2/db/create_db_op.cc",
"caffe2/db/protodb.cc",
"caffe2/share/contrib/depthwise/depthwise3x3_conv_op.cc",
":caffe2_core_srcs",
":caffe2_distributed_srcs",
":caffe2_ideep_srcs",
":caffe2_onnx_srcs",
":caffe2_operators_srcs",

5
CMakeLists.txt
View File

@ -230,9 +230,7 @@ option(USE_CUPTI_SO "Use CUPTI as a shared library" ON)
option(USE_FAKELOWP "Use FakeLowp operators" OFF)
option(USE_GFLAGS "Use GFLAGS" OFF)
option(USE_GLOG "Use GLOG" OFF)
option(USE_LEVELDB "Use LEVELDB" OFF)
option(USE_LITE_PROTO "Use lite protobuf instead of full." OFF)
option(USE_LMDB "Use LMDB" OFF)
option(USE_MAGMA "Use MAGMA" ON)
option(USE_METAL "Use Metal for Caffe2 iOS build" ON)
option(USE_PYTORCH_METAL "Use Metal for PyTorch iOS build" OFF)
@ -269,8 +267,6 @@ option(USE_PRECOMPILED_HEADERS "Use pre-compiled headers to accelerate build." O
option(USE_PROF "Use profiling" OFF)
option(USE_QNNPACK "Use QNNPACK (quantized 8-bit operators)" ON)
option(USE_PYTORCH_QNNPACK "Use ATen/QNNPACK (quantized 8-bit operators)" ON)
option(USE_REDIS "Use Redis" OFF)
option(USE_ROCKSDB "Use RocksDB" OFF)
option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
option(USE_SYSTEM_EIGEN_INSTALL
"Use system Eigen instead of the one under third_party" OFF)
@ -292,7 +288,6 @@ option(USE_VULKAN_FP16_INFERENCE "Vulkan - Use fp16 inference" OFF)
option(USE_VULKAN_RELAXED_PRECISION "Vulkan - Use relaxed precision math in the kernels (mediump)" OFF)
# option USE_XNNPACK: try to enable xnnpack by default.
option(USE_XNNPACK "Use XNNPACK" ON)
option(USE_ZMQ "Use ZMQ" OFF)
option(USE_ZSTD "Use ZSTD" OFF)
option(USE_ROCM_KERNEL_ASSERT "Use Kernel Assert for ROCm" OFF)
# Ensure that an ITT build is the default for x86 CPUs

43
binaries/CMakeLists.txt
View File

@ -8,20 +8,6 @@ if(INTERN_BUILD_MOBILE)
endif()
if(BUILD_CAFFE2)
caffe2_binary_target("convert_caffe_image_db.cc")
caffe2_binary_target("convert_db.cc")
caffe2_binary_target("make_cifar_db.cc")
caffe2_binary_target("make_mnist_db.cc")
caffe2_binary_target("predictor_verifier.cc")
caffe2_binary_target("speed_benchmark.cc")
caffe2_binary_target("split_db.cc")
caffe2_binary_target("print_registered_core_operators.cc")
if(USE_OBSERVERS)
caffe2_binary_target(caffe2_benchmark "caffe2_benchmark.cc" "benchmark_helper.cc")
target_include_directories(caffe2_benchmark PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../modules)
endif()
caffe2_binary_target("at_launch_benchmark.cc")
target_include_directories(at_launch_benchmark PUBLIC
${CMAKE_BINARY_DIR}/aten/src)
@ -29,12 +15,6 @@ if(BUILD_CAFFE2)
caffe2_binary_target("intra_inter_benchmark.cc")
target_include_directories(intra_inter_benchmark PUBLIC
${CMAKE_BINARY_DIR}/aten/src)
caffe2_binary_target("run_plan.cc")
caffe2_binary_target("db_throughput.cc")
# ---[ tutorials
caffe2_binary_target("tutorial_blob.cc")
endif()
caffe2_binary_target("parallel_info.cc")
@ -77,34 +57,11 @@ if(USE_ROCM)
endif()
endif()
if(USE_ZMQ)
caffe2_binary_target("zmq_feeder.cc")
target_link_libraries(zmq_feeder ${ZMQ_LIBRARIES})
endif()
if(USE_MPI)
caffe2_binary_target("run_plan_mpi.cc")
target_link_libraries(run_plan_mpi ${MPI_CXX_LIBRARIES})
endif()
if(USE_OPENCV AND USE_LEVELDB)
caffe2_binary_target("convert_encoded_to_raw_leveldb.cc")
target_link_libraries(
convert_encoded_to_raw_leveldb
${OpenCV_LIBS} ${LevelDB_LIBRARIES} ${Snappy_LIBRARIES})
endif()
if(USE_OPENCV)
caffe2_binary_target("make_image_db.cc")
target_link_libraries(make_image_db ${OpenCV_LIBS})
caffe2_binary_target("convert_image_to_tensor.cc")
target_link_libraries(convert_image_to_tensor ${OpenCV_LIBS})
endif()
if(USE_OBSERVERS AND USE_OPENCV)
caffe2_binary_target("convert_and_benchmark.cc")
target_link_libraries(convert_and_benchmark ${OpenCV_LIBS})
endif()
caffe2_binary_target("dump_operator_names.cc")
caffe2_binary_target("optimize_for_mobile.cc")

523
binaries/benchmark_helper.cc
View File

@ -1,523 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <chrono>
#include <fstream>
#include <iostream>
#include <string>
#include <thread>
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#include <psapi.h>
#endif
#include <binaries/benchmark_helper.h>
#include "caffe2/core/blob_serialization.h"
#ifdef __CUDA_ARCH__
#include "caffe2/core/context_gpu.h"
#endif
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/net.h"
#include "caffe2/core/operator.h"
#include "caffe2/core/tensor_int8.h"
#include "caffe2/utils/bench_utils.h"
#include "caffe2/utils/string_utils.h"
#include <observers/net_observer_reporter_print.h>
#include <observers/observer_config.h>
#include <observers/perf_observer.h>
#if defined(TARGET_OS_MAC) || \
defined(TARGET_OS_IPHONE) || \
defined(TARGET_IPHONE_SIMULATOR)
#include <malloc/malloc.h>
#else
#include <malloc.h>
#endif
void observerConfig() {
caffe2::ClearGlobalNetObservers();
caffe2::AddGlobalNetObserverCreator([](caffe2::NetBase* subject) {
return std::make_unique<caffe2::PerfNetObserver>(subject);
});
caffe2::ObserverConfig::setReporter(
std::make_unique<caffe2::NetObserverReporterPrint>());
}
bool backendCudaSet(const string& backend) {
bool run_on_gpu = false;
if (backend == "cuda") {
#ifdef __CUDA_ARCH__
if (caffe2::HasCudaGPU()) {
run_on_gpu = true;
} else {
CAFFE_THROW("NO GPU support on this host machine");
}
#else
CAFFE_THROW("NO GPU support");
#endif
}
return run_on_gpu;
}
void setDeviceType(caffe2::NetDef* net_def, caffe2::DeviceType& run_dev) {
for (int j = 0; j < net_def->op_size(); j++) {
caffe2::OperatorDef* op = net_def->mutable_op(j);
op->mutable_device_option()->set_device_type(caffe2::TypeToProto(run_dev));
}
}
void setOperatorEngine(caffe2::NetDef* net_def, const string& backend) {
if (backend != "builtin") {
string engine = backend == "nnpack"
? "NNPACK"
: backend == "eigen" ? "EIGEN"
: backend == "mkl" ? "MKLDNN"
: backend == "cuda"
? "CUDA"
: backend == "dnnlowp" ? "DNNLOWP"
: backend == "dnnlowp_acc16"
? "DNNLOWP_ACC16"
: backend == "default" ? "" : "NONE";
CAFFE_ENFORCE(engine != "NONE", "Backend is not supported");
for (int i = 0; i < net_def->op_size(); i++) {
caffe2::OperatorDef* op_def = net_def->mutable_op(i);
op_def->set_engine(engine);
}
}
}
int loadInput(
shared_ptr<caffe2::Workspace> workspace,
const bool run_on_gpu,
map<string, caffe2::TensorProtos>& tensor_protos_map,
const string& input,
const string& input_file,
const string& input_dims,
const string& input_type) {
// How many input blobs are in the inputs
int blob_num = 1;
// Load input.
if (input.size()) {
vector<string> input_names = caffe2::split(',', input);
if (input_file.size()) {
vector<string> input_files = caffe2::split(',', input_file);
CAFFE_ENFORCE_EQ(
input_names.size(),
input_files.size(),
"Input name and file should have the same number.");
for (int i = 0; i < input_names.size(); ++i) {
caffe2::TensorProtos tensor_protos;
CAFFE_ENFORCE(
caffe2::ReadProtoFromFile(input_files[i], &tensor_protos));
workspace->CreateBlob(input_names[i]);
tensor_protos_map.insert(std::make_pair(input_names[i], tensor_protos));
}
// Check that all blobs have the same number of entries
blob_num = tensor_protos_map[input_names[0]].protos_size();
for (int i = 1; i < input_names.size(); ++i) {
int bnum = tensor_protos_map[input_names[i]].protos_size();
CAFFE_ENFORCE_EQ(
blob_num,
bnum,
"Number of blobs are not the same for all inputs");
}
} else if (input_dims.size() || input_type.size()) {
CAFFE_ENFORCE_GE(
input_dims.size(),
0,
"Input dims must be specified when input tensors are used.");
CAFFE_ENFORCE_GE(
input_type.size(),
0,
"Input type must be specified when input tensors are used.");
vector<string> input_dims_list = caffe2::split(';', input_dims);
CAFFE_ENFORCE_EQ(
input_names.size(),
input_dims_list.size(),
"Input name and dims should have the same number of items.");
vector<string> input_type_list = caffe2::split(';', input_type);
CAFFE_ENFORCE_EQ(
input_names.size(),
input_type_list.size(),
"Input name and type should have the same number of items.");
for (size_t i = 0; i < input_names.size(); ++i) {
vector<string> input_dims_str = caffe2::split(',', input_dims_list[i]);
vector<int> input_dims;
for (const string& s : input_dims_str) {
input_dims.push_back(std::stoi(s));
}
caffe2::Blob* blob = workspace->GetBlob(input_names[i]);
if (blob == nullptr) {
blob = workspace->CreateBlob(input_names[i]);
}
if (run_on_gpu) {
LOG(INFO) << "Running on GPU.";
#ifdef __CUDA_ARCH__
caffe2::TensorCUDA* tensor = blob->GetMutable<caffe2::TensorCUDA>();
TORCH_CHECK_NOTNULL(tensor);
tensor->Resize(input_dims);
if (input_type_list[i] == "uint8_t") {
tensor->mutable_data<uint8_t>();
} else if (input_type_list[i] == "float") {
tensor->mutable_data<float>();
} else {
CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
}
#else
CAFFE_THROW("Not support GPU on mobile.");
#endif
} else {
if (input_type_list[i] == "uint8_t") {
caffe2::int8::Int8TensorCPU* tensor =
blob->GetMutable<caffe2::int8::Int8TensorCPU>();
TORCH_CHECK_NOTNULL(tensor);
tensor->t.Resize(input_dims);
tensor->t.mutable_data<uint8_t>();
} else if (input_type_list[i] == "float") {
caffe2::TensorCPU* tensor = BlobGetMutableTensor(blob, caffe2::CPU);
TORCH_CHECK_NOTNULL(tensor);
tensor->Resize(input_dims);
tensor->mutable_data<float>();
} else if (input_type_list[i] == "int") {
caffe2::TensorCPU* tensor = BlobGetMutableTensor(blob, caffe2::CPU);
TORCH_CHECK_NOTNULL(tensor);
tensor->Resize(input_dims);
tensor->mutable_data<int>();
} else {
CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
}
}
}
} else {
CAFFE_THROW(
"You requested input tensors, but neither input_file nor "
"input_dims is set.");
}
}
return blob_num;
}
void fillInputBlob(
shared_ptr<caffe2::Workspace> workspace,
map<string, caffe2::TensorProtos>& tensor_protos_map,
int iteration) {
if (tensor_protos_map.empty()) {
return;
}
static caffe2::TensorDeserializer deserializer;
for (auto& tensor_kv : tensor_protos_map) {
caffe2::Blob* blob = workspace->GetBlob(tensor_kv.first);
if (blob == nullptr) {
blob = workspace->CreateBlob(tensor_kv.first);
}
// todo: support gpu and make this function a template
int protos_size = tensor_kv.second.protos_size();
if (protos_size == 1 && iteration > 0) {
// Do not override the input data if there is only one input data,
// since it will clear all caches. Rely on wipe_cache to
// clear caches
continue;
}
caffe2::TensorProto* tensor_proto =
tensor_kv.second.mutable_protos(iteration % protos_size);
BlobSetTensor(blob, deserializer.Deserialize(*tensor_proto));
// todo: for other types
}
}
void runNetwork(
shared_ptr<caffe2::Workspace> workspace,
caffe2::NetBase* net,
map<string, caffe2::TensorProtos>& tensor_protos_map,
const bool wipe_cache,
const bool run_individual,
const bool run_on_gpu,
const bool text_output,
const int warmup,
const int iter,
const int num_blobs,
const int sleep_before_run,
const int sleep_between_iteration,
const int sleep_between_net_and_operator,
const std::string& output,
const std::string& output_folder) {
LOG(INFO) << "Starting benchmark.";
caffe2::ObserverConfig::initSampleRate(1, 1, 1, run_individual, warmup);
LOG(INFO) << "Running warmup runs.";
for (int i = 0; i < warmup; ++i) {
fillInputBlob(workspace, tensor_protos_map, i);
CAFFE_ENFORCE(net->Run(), "Warmup run ", i, " has failed.");
}
if (wipe_cache) {
caffe2::wipe_cache();
}
if (sleep_before_run > 0) {
std::this_thread::sleep_for(std::chrono::seconds(sleep_before_run));
}
LOG(INFO) << "Main runs.";
CAFFE_ENFORCE(
iter >= 0,
"Number of main runs should be non negative, provided ",
iter,
".");
LOG(INFO) << "net runs.";
long long duration_sum = 0;
for (int i = 0; i < iter; ++i) {
caffe2::ObserverConfig::initSampleRate(1, 1, 1, 0, warmup);
fillInputBlob(workspace, tensor_protos_map, i);
if (wipe_cache) {
caffe2::wipe_cache();
}
auto start = std::chrono::high_resolution_clock::now();
CAFFE_ENFORCE(net->Run(), "Main run ", i, " has failed.");
auto stop = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(stop - start);
duration_sum += duration.count();
// Write the output for the first num_blobs times
writeOutput(
workspace,
run_on_gpu,
output,
output_folder,
text_output,
i,
num_blobs);
if (wipe_cache) {
caffe2::wipe_cache();
}
if (sleep_between_iteration > 0) {
std::this_thread::sleep_for(
std::chrono::seconds(sleep_between_iteration));
}
}
std::cout << "Average Duration: " << (duration_sum/iter) << " us" << std::endl;
if (run_individual) {
LOG(INFO) << "operator runs.";
if (sleep_between_net_and_operator > 0) {
std::this_thread::sleep_for(
std::chrono::seconds(sleep_between_net_and_operator));
}
for (int i = 0; i < iter; ++i) {
caffe2::ObserverConfig::initSampleRate(1, 1, 1, 1, warmup);
fillInputBlob(workspace, tensor_protos_map, i);
CAFFE_ENFORCE(net->Run(), "Main run ", i, " with operator has failed.");
if (wipe_cache) {
caffe2::wipe_cache();
}
if (sleep_between_iteration > 0) {
std::this_thread::sleep_for(
std::chrono::seconds(sleep_between_iteration));
}
}
}
}
void writeOutput(
shared_ptr<caffe2::Workspace> workspace,
const bool run_on_gpu,
const string& output,
const string& output_folder,
const bool text_output,
const int index,
const int num_blobs) {
if (output.size() == 0) {
return;
}
string output_prefix = output_folder.size() ? output_folder + "/" : "";
vector<string> output_names = caffe2::split(',', output);
if (output == "*") {
output_names = workspace->Blobs();
}
for (const string& name : output_names) {
CAFFE_ENFORCE(
workspace->HasBlob(name),
"You requested a non-existing blob: ",
name);
if (text_output) {
if (run_on_gpu) {
#ifdef __CUDA_ARCH__
writeTextOutput<caffe2::CUDAContext, caffe2::TensorCUDA>(
workspace->GetBlob(name)->GetMutable<caffe2::TensorCUDA>(),
output_prefix,
name,
index,
num_blobs);
#else
CAFFE_THROW("Not support GPU.");
#endif
} else {
writeTextOutput<caffe2::CPUContext, caffe2::TensorCPU>(
BlobGetMutableTensor(workspace->GetBlob(name), caffe2::CPU),
output_prefix,
name,
index,
num_blobs);
}
} else {
// Do not support multiple entries per blob.
CAFFE_ENFORCE(
index == 0,
"Binary file only support one output.");
string serialized = SerializeBlob(*workspace->GetBlob(name), name);
string output_filename = output_prefix + name;
caffe2::WriteStringToFile(serialized, output_filename.c_str());
}
}
}
void logBenchmarkResult(
const std::string& type,
const std::string& metric,
const std::string& unit,
const int value) {
LOG(INFO) << caffe2::NetObserverReporterPrint::IDENTIFIER << "{"
<< "\"type\": \"" << type << "\", "
<< "\"metric\": \"" << metric << "\", "
<< "\"unit\": \"" << unit << "\", "
<< "\"value\": " << c10::to_string(value) << "}\n";
}
long getVirtualMemoryIfOptionEnabled(bool FLAGS_measure_memory) {
if (FLAGS_measure_memory) {
#if defined(TARGET_OS_IPHONE) || \
defined(TARGET_OS_MAC) || \
defined(TARGET_IPHONE_SIMULATOR)
malloc_statistics_t stats = {0};
malloc_zone_statistics(nullptr, &stats);
return stats.size_allocated;
#elif defined(_WIN32)
PROCESS_MEMORY_COUNTERS_EX pmc;
GetProcessMemoryInfo(
GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc));
return pmc.PrivateUsage;
#else
struct mallinfo info = mallinfo();
return info.uordblks;
#endif
}
return 0;
}
int benchmark(
int argc,
char* argv[],
const string& FLAGS_backend,
const string& FLAGS_init_net,
const string& FLAGS_input,
const string& FLAGS_input_dims,
const string& FLAGS_input_file,
const string& FLAGS_input_type,
int FLAGS_iter,
bool FLAGS_measure_memory,
const string& FLAGS_net,
const string& FLAGS_output,
const string& FLAGS_output_folder,
bool FLAGS_run_individual,
int FLAGS_sleep_before_run,
int FLAGS_sleep_between_iteration,
int FLAGS_sleep_between_net_and_operator,
bool FLAGS_text_output,
int FLAGS_warmup,
bool FLAGS_wipe_cache) {
// Check arguments to be correct
{
// Need to check whether file exists, as the file reader does not assert if
// file does not exist
std::ifstream net_file(FLAGS_net);
CAFFE_ENFORCE(net_file.good());
net_file.close();
std::ifstream init_net_file(FLAGS_init_net);
CAFFE_ENFORCE(init_net_file.good());
init_net_file.close();
if (FLAGS_input_file.size() > 0) {
vector<string> input_files = caffe2::split(',', FLAGS_input_file);
for (auto input_file : input_files) {
std::ifstream ifile(input_file);
CAFFE_ENFORCE(ifile.good());
ifile.close();
}
}
}
observerConfig();
caffe2::ShowLogInfoToStderr();
auto workspace = std::make_shared<caffe2::Workspace>(new caffe2::Workspace());
bool run_on_gpu = backendCudaSet(FLAGS_backend);
// Run initialization network, measure resources used.
long init_vmem = getVirtualMemoryIfOptionEnabled(FLAGS_measure_memory);
caffe2::NetDef init_net_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net_def));
setOperatorEngine(&init_net_def, FLAGS_backend);
CAFFE_ENFORCE(workspace->RunNetOnce(init_net_def));
init_vmem = getVirtualMemoryIfOptionEnabled(FLAGS_measure_memory) - init_vmem;
map<string, caffe2::TensorProtos> tensor_protos_map;
int num_blobs = loadInput(
workspace,
run_on_gpu,
tensor_protos_map,
FLAGS_input,
FLAGS_input_file,
FLAGS_input_dims,
FLAGS_input_type);
// Run main network.
long predict_vmem = getVirtualMemoryIfOptionEnabled(FLAGS_measure_memory);
caffe2::NetDef net_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_net, &net_def));
setOperatorEngine(&net_def, FLAGS_backend);
if (!net_def.has_name()) {
net_def.set_name("benchmark");
}
caffe2::NetBase* net = workspace->CreateNet(net_def);
TORCH_CHECK_NOTNULL(net);
runNetwork(
workspace,
net,
tensor_protos_map,
FLAGS_wipe_cache,
FLAGS_run_individual,
run_on_gpu,
FLAGS_text_output,
FLAGS_warmup,
FLAGS_iter,
num_blobs,
FLAGS_sleep_before_run,
FLAGS_sleep_between_iteration,
FLAGS_sleep_between_net_and_operator,
FLAGS_output,
FLAGS_output_folder);
predict_vmem = getVirtualMemoryIfOptionEnabled(
FLAGS_measure_memory) - predict_vmem;
if (FLAGS_measure_memory) {
logBenchmarkResult(
"NET_", "memory", "kB", (init_vmem + predict_vmem) / 1024);
}
return 0;
}

169
binaries/benchmark_helper.h
View File

@ -1,169 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <string>
#include "caffe2/core/blob_serialization.h"
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/net.h"
#include "caffe2/core/operator.h"
#include "caffe2/utils/string_utils.h"
#include "c10/util/string_utils.h"
#include <c10/util/irange.h>
using std::map;
using std::shared_ptr;
using std::string;
using std::vector;
template <typename ContextType, typename TensorType>
void writeTextOutput(
TensorType* tensor,
const string& output_prefix,
const string& name,
int index,
int num_blobs) {
if (index >= num_blobs) {
return;
}
string filename = name;
std::replace(filename.begin(), filename.end(), '/', '_');
string output_name = output_prefix + "/" + filename + ".txt";
caffe2::TensorSerializer ser;
caffe2::BlobProto blob_proto;
ser.Serialize(
*tensor, output_name, blob_proto.mutable_tensor(), 0, tensor->numel());
blob_proto.set_name(output_name);
blob_proto.set_type("Tensor");
CAFFE_ENFORCE(blob_proto.has_tensor());
caffe2::TensorProto tensor_proto = blob_proto.tensor();
int dims_size = tensor_proto.dims_size();
long long elem_dim_size =
dims_size > 1 ? tensor_proto.dims(1) : tensor_proto.dims(0);
for (const auto i : c10::irange(2, dims_size)) {
elem_dim_size *= tensor_proto.dims(i);
}
std::vector<std::string> lines;
std::string dims;
for (const auto i : c10::irange(dims_size)) {
int dim = tensor_proto.dims(i);
if (i > 0) {
dims += ", ";
}
dims += c10::to_string(dim);
}
lines.push_back(dims);
std::stringstream line;
if (tensor_proto.data_type() == caffe2::TensorProto::FLOAT) {
auto start = tensor_proto.float_data().begin();
auto end = tensor_proto.float_data().end();
copy(start, end, std::ostream_iterator<float>(line, ","));
} else if (tensor_proto.data_type() == caffe2::TensorProto::INT32) {
auto start = tensor_proto.int32_data().begin();
auto end = tensor_proto.int32_data().end();
copy(start, end, std::ostream_iterator<int>(line, ","));
} else {
CAFFE_THROW("Unimplemented Blob type.");
}
// remove the last ,
string str = line.str();
if(str.length() != 0) {
str.pop_back();
}
lines.push_back(str);
// static casts are workaround for MSVC build
auto flags = static_cast<std::ios_base::openmode>(std::ios::out);
if (index != 0) {
flags |= static_cast<std::ios_base::openmode>(std::ios::app);
} else {
flags |= static_cast<std::ios_base::openmode>(std::ios::trunc);
}
std::ofstream output_file(output_name, flags);
std::ostream_iterator<std::string> output_iterator(output_file, "\n");
std::copy(lines.begin(), lines.end(), output_iterator);
}
void observerConfig();
bool backendCudaSet(const string&);
void setDeviceType(caffe2::NetDef*, caffe2::DeviceType&);
void setOperatorEngine(caffe2::NetDef*, const string&);
int loadInput(
shared_ptr<caffe2::Workspace> workspace,
const bool run_on_gpu,
map<string, caffe2::TensorProtos>& tensor_protos_map,
const string& input,
const string& input_file,
const string& input_dims,
const string& input_type);
void fillInputBlob(
shared_ptr<caffe2::Workspace> workspace,
map<string, caffe2::TensorProtos>& tensor_protos_map,
int iteration);
void writeOutput(
shared_ptr<caffe2::Workspace> workspace,
const bool run_on_gpu,
const string& output,
const string& output_folder,
const bool text_output,
const int index,
const int num_blobs);
void logBenchmarkResult(
const std::string& type,
const std::string& metric,
const std::string& unit,
const int value);
long getVirtualMemoryIfOptionEnabled(bool FLAGS_measure_memory);
void runNetwork(
shared_ptr<caffe2::Workspace> workspace,
caffe2::NetBase* net,
map<string, caffe2::TensorProtos>& tensor_protos_map,
const bool wipe_cache,
const bool run_individual,
const bool run_on_gpu,
const bool text_output,
const int warmup,
const int iter,
const int num_blobs,
const int sleep_before_run,
const int sleep_between_iteration,
const int sleep_between_net_and_operator,
const std::string& output,
const std::string& output_folder);
int benchmark(
int argc,
char* argv[],
const string& FLAGS_backend,
const string& FLAGS_init_net,
const string& FLAGS_input,
const string& FLAGS_input_dims,
const string& FLAGS_input_file,
const string& FLAGS_input_type,
int FLAGS_iter,
bool FLAGS_measure_memory,
const string& FLAGS_net,
const string& FLAGS_output,
const string& FLAGS_output_folder,
bool FLAGS_run_individual,
int FLAGS_sleep_before_run,
int FLAGS_sleep_between_iteration,
int FLAGS_sleep_between_net_and_operator,
bool FLAGS_text_output,
int FLAGS_warmup,
bool FLAGS_wipe_cache);

32
binaries/caffe2_benchmark.cc
View File

@ -1,32 +0,0 @@
#include <fstream>
#include <iterator>
#include <string>
#include "binaries/benchmark_args.h"
#include "binaries/benchmark_helper.h"
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
benchmark(
argc,
argv,
FLAGS_backend,
FLAGS_init_net,
FLAGS_input,
FLAGS_input_dims,
FLAGS_input_file,
FLAGS_input_type,
FLAGS_iter,
FLAGS_measure_memory,
FLAGS_net,
FLAGS_output,
FLAGS_output_folder,
FLAGS_run_individual,
FLAGS_sleep_before_run,
FLAGS_sleep_between_iteration,
FLAGS_sleep_between_net_and_operator,
FLAGS_text_output,
FLAGS_warmup,
FLAGS_wipe_cache);
}

770
binaries/convert_and_benchmark.cc
View File

@ -1,770 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <opencv2/opencv.hpp>
#include <cmath>
#include <fstream>
#include "caffe2/core/common.h"
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/timer.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/utils/proto_utils.h"
#include "caffe2/utils/string_utils.h"
#include "caffe2/utils/bench_utils.h"
#include "binaries/benchmark_args.h"
#include "binaries/benchmark_helper.h"
#include <observers/net_observer_reporter_print.h>
#include <observers/observer_config.h>
#include <observers/perf_observer.h>
C10_DEFINE_int(
batch_size,
-1,
"Specify the batch size of the input. The number of items in the "
"input needs to be multiples of the batch size. If the batch size "
"is less than 0, all inputs are in one batch.")
C10_DEFINE_bool(color, true, "If set, load images in color.");
C10_DEFINE_string(
crop,
"-1,-1",
"The center cropped hight and width. If the value is less than zero, "
"it is not cropped.");
C10_DEFINE_string(input_image_files, "", "Files containing imput images");
C10_DEFINE_string(input_text_files, "", "Text files to be written to blobs");
C10_DEFINE_string(
preprocess,
"",
"Options to specify the preprocess routines. The available options are "
"subtract128, normalize, mean, std, bgrtorgb. If multiple steps are provided, they "
"are separated by comma (,) in sequence.");
C10_DEFINE_string(
report_time,
"",
"Report the conversion stage time to screen. "
"The format of the string is <type>|<identifier>. "
"The valid type is 'json'. "
"The valid identifier is nothing or an identifier that prefix every line");
C10_DEFINE_string(
scale,
"-1,-1",
"Scale the images to be within the min,max box. The shorter edge is "
"min pixels. But if the other edge is more than the max pixels, the "
"other edge and scaled to max pixels (and the shorter edge can be less "
"than the min pixels");
C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
namespace caffe2 {
void reportTime(
std::string type,
double ts,
std::string metric,
std::string unit) {
if (FLAGS_report_time == "") {
return;
}
vector<string> s = caffe2::split('|', FLAGS_report_time);
assert(s[0] == "json");
std::string identifier = "";
if (s.size() > 1) {
identifier = s[1];
}
std::cout << identifier << "{\"type\": \"" << type << "\", \"value\": " << ts
<< ", \"metric\": \"" << metric << "\", \"unit\": \"" << unit
<< "\"}" << std::endl;
}
void splitSizes(const std::string& arg, int* ptr0, int* ptr1) {
vector<string> sizes = caffe2::split(',', arg);
if (sizes.size() == 2) {
*ptr0 = std::stoi(sizes[0]);
*ptr1 = std::stoi(sizes[1]);
} else if (sizes.size() == 1) {
*ptr0 = std::stoi(sizes[0]);
*ptr1 = std::stoi(sizes[0]);
} else {
assert(false);
}
}
cv::Mat resizeImage(cv::Mat& img) {
int min_size, max_size;
splitSizes(FLAGS_scale, &min_size, &max_size);
if ((min_size <= 0) && (max_size <= 0)) {
return img;
}
if (max_size < 0) {
max_size = INT_MAX;
}
assert(min_size <= max_size);
int im_min_size = img.rows > img.cols ? img.cols : img.rows;
int im_max_size = img.rows > img.cols ? img.rows : img.cols;
double im_scale = 1.0 * min_size / im_min_size;
if (im_scale * im_max_size > max_size) {
im_scale = 1.0 * max_size / im_max_size;
}
int scaled_width = int(round(img.cols * im_scale));
int scaled_height = int(round(img.rows * im_scale));
assert((scaled_width <= max_size) && (scaled_height <= max_size));
if ((scaled_width < min_size) || (scaled_height < min_size)) {
assert((scaled_width == max_size) || (scaled_height == max_size));
} else {
assert((scaled_width == min_size) || (scaled_height == min_size));
}
cv::Mat resized_img;
cv::resize(
img,
resized_img,
cv::Size(),
im_scale,
im_scale,
cv::INTER_LINEAR);
return resized_img;
}
cv::Mat cropToRec(cv::Mat& img, int* height_ptr, int* width_ptr) {
int height = *height_ptr;
int width = *width_ptr;
if ((height > 0) && (width > 0) &&
((img.rows != height) || (img.cols != width))) {
cv::Mat cropped_img, cimg;
cv::Rect roi;
roi.x = int((img.cols - width) / 2);
roi.y = int((img.rows - height) / 2);
roi.x = roi.x < 0 ? 0 : roi.x;
roi.y = roi.y < 0 ? 0 : roi.y;
width = width > img.cols ? img.cols : width;
height = height > img.rows ? img.rows : height;
roi.width = width;
roi.height = height;
assert(
0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= img.cols &&
0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= img.rows);
cropped_img = img(roi);
// Make the image in continuous space in memory
cimg = cropped_img.clone();
*height_ptr = height;
*width_ptr = width;
return cimg;
} else {
return img;
}
}
std::vector<float> convertToVector(cv::Mat& img) {
std::vector<float> normalize(3, 1);
std::vector<float> mean(3, 0);
std::vector<float> std(3, 1);
bool bgrtorgb = false;
int size = img.cols * img.rows;
vector<string> steps = caffe2::split(',', FLAGS_preprocess);
for (int i = 0; i < steps.size(); i++) {
auto step = steps[i];
if (step == "subtract128") {
mean = {128, 128, 128};
std = {1, 1, 1};
normalize = {1, 1, 1};
} else if (step == "normalize") {
normalize = {255, 255, 255};
} else if (step == "mean") {
mean = {0.406f, 0.456f, 0.485f};
} else if (step == "std") {
std = {0.225f, 0.224f, 0.229f};
} else if (step == "bgrtorgb") {
bgrtorgb = true;
} else {
CAFFE_ENFORCE(
false,
"Unsupported preprocess step. The supported steps are: subtract128, "
"normalize,mean, std, swaprb.");
}
}
int C = FLAGS_color ? 3 : 1;
int total_size = C * size;
std::vector<float> values(total_size);
if (C == 1) {
cv::MatIterator_<float> it, end;
int idx = 0;
for (it = img.begin<float>(), end = img.end<float>(); it != end; ++it) {
values[idx++] = (*it / normalize[0] - mean[0]) / std[0];
}
} else {
int i = 0;
cv::MatIterator_<cv::Vec3f> it, end;
int b = bgrtorgb ? 2 : 0;
int g = 1;
int r = bgrtorgb ? 0 : 2;
for (it = img.begin<cv::Vec3f>(), end = img.end<cv::Vec3f>(); it != end;
++it, i++) {
values[i] = (((*it)[b] / normalize[0] - mean[0]) / std[0]);
int offset = size + i;
values[offset] = (((*it)[g] / normalize[1] - mean[1]) / std[1]);
offset = size + offset;
values[offset] = (((*it)[r] / normalize[2] - mean[2]) / std[2]);
}
}
return values;
}
std::vector<float> convertOneImage(
std::string& filename,
int* height_ptr,
int* width_ptr) {
assert(filename[0] != '~');
std::cout << "Converting " << filename << std::endl;
// Load image
cv::Mat img_uint8 = cv::imread(
#if CV_MAJOR_VERSION <= 3
filename, FLAGS_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);
#else
filename, FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
#endif
caffe2::Timer timer;
timer.Start();
cv::Mat img;
// Convert image to floating point values
img_uint8.convertTo(img, CV_32F);
// Resize image
cv::Mat resized_img = resizeImage(img);
int height, width;
splitSizes(FLAGS_crop, &height, &width);
if ((height <= 0) || (width <= 0)) {
height = resized_img.rows;
width = resized_img.cols;
}
cv::Mat crop = cropToRec(resized_img, &height, &width);
// Assert we don't have to deal with alignment
DCHECK(crop.isContinuous());
assert(crop.rows == height);
assert(crop.cols == width);
std::vector<float> one_image_values = convertToVector(crop);
*height_ptr = height;
*width_ptr = width;
double ts = timer.MicroSeconds();
reportTime("image_preprocess", ts, "convert", "us");
return one_image_values;
}
int getBatchSize(int num_items) {
int batch_size = FLAGS_batch_size;
if (batch_size < 0) {
batch_size = num_items;
} else {
assert(num_items % batch_size == 0);
}
return batch_size;
}
TensorProtos writeValues(
std::vector<std::vector<std::vector<float>>>& values,
std::vector<std::vector<int>>& dims) {
caffe2::Timer timer;
timer.Start();
assert(dims.size() == values.size());
int num_batches = dims.size();
TensorProtos protos;
for (int k = 0; k < num_batches; k++) {
TensorProto* data;
data = protos.add_protos();
data->set_data_type(TensorProto::FLOAT);
auto one_dim = dims[k];
for (int dim : one_dim) {
data->add_dims(dim);
}
int batch_size = one_dim[0];
long long int entry_size = 1;
for (int i = 1; i < one_dim.size(); i++) {
entry_size *= one_dim[i];
}
// Not optimized
for (int i = 0; i < batch_size; i++) {
assert(values[k][i].size() == entry_size);
for (int j = 0; j < values[k][i].size(); j++) {
data->add_float_data(values[k][i][j]);
}
}
}
double ts = timer.MicroSeconds();
reportTime("preprocess", ts, "data_pack", "us");
return protos;
}
TensorProtos convertImages(std::string& image_file) {
vector<string> file_names;
if (image_file != "") {
std::ifstream infile(image_file);
std::string line;
while (std::getline(infile, line)) {
vector<string> file_name = caffe2::split(',', line);
string name;
if (file_name.size() == 3) {
name = file_name[2];
} else {
name = line;
}
file_names.push_back(name);
}
} else {
TensorProtos proto;
return proto;
}
int batch_size = getBatchSize(file_names.size());
int num_batches = file_names.size() / batch_size;
assert(file_names.size() == batch_size * num_batches);
std::vector<std::vector<std::vector<float>>> values;
std::vector<std::vector<int>> dims;
int C = FLAGS_color ? 3 : 1;
for (int k = 0; k < num_batches; k++) {
std::vector<std::vector<float>> one_value;
int height = -1;
int width = -1;
for (int i = 0; i < batch_size; i++) {
int idx = k * batch_size + i;
int one_height, one_width;
std::vector<float> one_image_values =
convertOneImage(file_names[idx], &one_height, &one_width);
if (height < 0 && width < 0) {
height = one_height;
width = one_width;
} else {
assert(height == one_height);
assert(width == one_width);
}
one_value.push_back(one_image_values);
}
vector<int> one_dim = {batch_size, C, height, width};
dims.push_back(one_dim);
values.push_back(one_value);
}
return writeValues(values, dims);
}
template <class TYPE>
vector<TYPE> splitString(std::string& line) {
vector<string> vector_str = caffe2::split(',', line);
vector<TYPE> vector_int;
for (string str : vector_str) {
vector_int.push_back((TYPE)std::stod(str));
}
return vector_int;
}
/* Convert the values in a json file to blobs
The format of the json file should be:
<number of items>, <dim2>.... (dimensions of items)
<entry>, <entry>, <entry>... (all entries in one item)
<entry>, <entry>, <entry>...
....
*/
TensorProtos convertValues(std::string& file_name) {
if (file_name == "") {
TensorProtos proto;
return proto;
}
std::ifstream infile(file_name);
std::string line;
std::getline(infile, line);
vector<int> file_dims = splitString <int>(line);
assert(file_dims.size() >= 2);
int num_items = file_dims[0];
int batch_size = getBatchSize(num_items);
int num_batches = num_items / batch_size;
assert(num_items == batch_size * num_batches);
vector<string> lines;
while (std::getline(infile, line)) {
lines.push_back(line);
}
assert(lines.size() == num_items);
std::vector<std::vector<std::vector<float>>> values;
std::vector<std::vector<int>> dims;
for (int i = 0; i < num_batches; i++) {
std::vector<std::vector<float>> one_value;
int num = -1;
for (int j = 0; j < batch_size; j++) {
int idx = i * batch_size + j;
std::string line = lines[idx];
vector<float> item = splitString<float>(line);
if (num < 0) {
num = item.size();
} else {
assert(num == item.size());
}
one_value.push_back(item);
}
vector<int> batch_dims = file_dims;
batch_dims[0] = batch_size;
dims.push_back(batch_dims);
values.push_back(one_value);
}
return writeValues(values, dims);
}
} // namespace caffe2
void observerConfig() {
caffe2::ClearGlobalNetObservers();
caffe2::AddGlobalNetObserverCreator([](caffe2::NetBase* subject) {
return std::make_unique<caffe2::PerfNetObserver>(subject);
});
caffe2::ObserverConfig::setReporter(
std::make_unique<caffe2::NetObserverReporterPrint>());
}
bool backendCudaSet(const string& backend) {
bool run_on_gpu = false;
if (backend == "cuda") {
#ifdef __CUDA_ARCH__
if (caffe2::HasCudaGPU()) {
run_on_gpu = true;
} else {
CAFFE_THROW("NO GPU support on this host machine");
}
#else
CAFFE_THROW("NO GPU support");
#endif
}
return run_on_gpu;
}
void setOperatorEngine(caffe2::NetDef* net_def, const string& backend) {
if (backend != "builtin") {
string engine;
if( backend == "nnpack" ) {
engine = "NNPACK";
} else if ( backend == "eigen" ) {
engine = "EIGEN";
} else if ( backend == "mkl" ) {
engine = "MKLDNN";
} else if ( backend == "cuda" ) {
engine = "CUDA";
} else if ( backend == "dnnlowp" ) {
engine = "DNNLOWP";
} else if ( backend == "dnnlowp_acc16" ) {
engine = "DNNLOWP_ACC16";
} else if ( backend == "default" ) {
engine = "";
} else {
engine = "NONE";
}
CAFFE_ENFORCE(engine != "NONE", "Backend is not supported");
for (int i = 0; i < net_def->op_size(); i++) {
caffe2::OperatorDef* op_def = net_def->mutable_op(i);
op_def->set_engine(engine);
}
}
}
void fillInputBlob(
shared_ptr<caffe2::Workspace> workspace,
map<string, caffe2::TensorProtos>& tensor_protos_map,
int iteration) {
if (tensor_protos_map.empty()) {
return;
}
static caffe2::TensorDeserializer deserializer;
for (auto& tensor_kv : tensor_protos_map) {
caffe2::Blob* blob = workspace->GetBlob(tensor_kv.first);
if (blob == nullptr) {
blob = workspace->CreateBlob(tensor_kv.first);
}
// todo: support gpu and make this function a template
int protos_size = tensor_kv.second.protos_size();
if (protos_size == 1 && iteration > 0) {
// Do not override the input data if there is only one input data,
// since it will clear all caches. Rely on wipe_cache to
// clear caches
continue;
}
caffe2::TensorProto* tensor_proto =
tensor_kv.second.mutable_protos(iteration % protos_size);
BlobSetTensor(blob, deserializer.Deserialize(*tensor_proto));
// todo: for other types
}
}
void writeOutput(
shared_ptr<caffe2::Workspace> workspace,
const bool run_on_gpu,
const string& output,
const string& output_folder,
const bool text_output,
const int index,
const int num_blobs) {
if (output.size() == 0) {
return;
}
string output_prefix = output_folder.size() ? output_folder + "/" : "";
vector<string> output_names = caffe2::split(',', output);
if (output == "*") {
output_names = workspace->Blobs();
}
for (const string& name : output_names) {
CAFFE_ENFORCE(
workspace->HasBlob(name),
"You requested a non-existing blob: ",
name);
if (text_output) {
if (run_on_gpu) {
#ifdef __CUDA_ARCH__
writeTextOutput<caffe2::CUDAContext, caffe2::TensorCUDA>(
workspace->GetBlob(name)->GetMutable<caffe2::TensorCUDA>(),
output_prefix,
name,
index,
num_blobs);
#else
CAFFE_THROW("Not support GPU.");
#endif
} else {
writeTextOutput<caffe2::CPUContext, caffe2::TensorCPU>(
BlobGetMutableTensor(workspace->GetBlob(name), caffe2::CPU),
output_prefix,
name,
index,
num_blobs);
}
} else {
// Do not support multiple entries per blob.
CAFFE_ENFORCE(
index == 0,
"Binary file only support one output.");
string serialized = SerializeBlob(*workspace->GetBlob(name), name);
string output_filename = output_prefix + name;
caffe2::WriteStringToFile(serialized, output_filename.c_str());
}
}
}
void runNetwork(
shared_ptr<caffe2::Workspace> workspace,
caffe2::NetDef& net_def,
map<string, caffe2::TensorProtos>& tensor_protos_map,
const bool wipe_cache,
const bool run_individual,
const bool run_on_gpu,
const bool text_output,
const int warmup,
const int iter,
const int num_blobs,
const int sleep_before_run,
const int sleep_between_iteration,
const int sleep_between_net_and_operator,
const std::string& output,
const std::string& output_folder) {
if (!net_def.has_name()) {
net_def.set_name("benchmark");
}
caffe2::NetBase* net = workspace->CreateNet(net_def);
TORCH_CHECK_NOTNULL(net);
LOG(INFO) << "Starting benchmark.";
caffe2::ObserverConfig::initSampleRate(1, 1, 1, run_individual, warmup);
LOG(INFO) << "Running warmup runs.";
for (int i = 0; i < warmup; ++i) {
fillInputBlob(workspace, tensor_protos_map, i);
CAFFE_ENFORCE(net->Run(), "Warmup run ", i, " has failed.");
}
if (wipe_cache) {
caffe2::wipe_cache();
}
if (sleep_before_run > 0) {
std::this_thread::sleep_for(std::chrono::seconds(sleep_before_run));
}
LOG(INFO) << "Main runs.";
CAFFE_ENFORCE(
iter >= 0,
"Number of main runs should be non negative, provided ",
iter,
".");
LOG(INFO) << "net runs.";
for (int i = 0; i < iter; ++i) {
caffe2::ObserverConfig::initSampleRate(1, 1, 1, 0, warmup);
fillInputBlob(workspace, tensor_protos_map, i);
if (wipe_cache) {
caffe2::wipe_cache();
}
CAFFE_ENFORCE(net->Run(), "Main run ", i, " has failed.");
// Write the output for the first num_blobs times
writeOutput(
workspace,
run_on_gpu,
output,
output_folder,
text_output,
i,
num_blobs);
if (wipe_cache) {
caffe2::wipe_cache();
}
if (sleep_between_iteration > 0) {
std::this_thread::sleep_for(
std::chrono::seconds(sleep_between_iteration));
}
}
if (run_individual) {
LOG(INFO) << "operator runs.";
if (sleep_between_net_and_operator > 0) {
std::this_thread::sleep_for(
std::chrono::seconds(sleep_between_net_and_operator));
}
for (int i = 0; i < iter; ++i) {
caffe2::ObserverConfig::initSampleRate(1, 1, 1, 1, warmup);
fillInputBlob(workspace, tensor_protos_map, i);
CAFFE_ENFORCE(net->Run(), "Main run ", i, " with operator has failed.");
if (wipe_cache) {
caffe2::wipe_cache();
}
if (sleep_between_iteration > 0) {
std::this_thread::sleep_for(
std::chrono::seconds(sleep_between_iteration));
}
}
}
}
int benchmark(
int argc,
char* argv[],
const string& FLAGS_backend,
const string& FLAGS_init_net,
const string& FLAGS_input_dims,
int FLAGS_iter,
const string& FLAGS_net,
const string& FLAGS_output,
const string& FLAGS_output_folder,
bool FLAGS_run_individual,
int FLAGS_sleep_before_run,
int FLAGS_sleep_between_iteration,
int FLAGS_sleep_between_net_and_operator,
bool FLAGS_text_output,
int FLAGS_warmup,
bool FLAGS_wipe_cache) {
// Check arguments to be correct
{
// Need to check whether file exists, as the file reader does not assert if
// file does not exist
std::ifstream net_file(FLAGS_net);
CAFFE_ENFORCE(net_file.good());
net_file.close();
std::ifstream init_net_file(FLAGS_init_net);
CAFFE_ENFORCE(init_net_file.good());
init_net_file.close();
}
observerConfig();
caffe2::ShowLogInfoToStderr();
auto workspace = std::make_shared<caffe2::Workspace>(new caffe2::Workspace());
bool run_on_gpu = backendCudaSet(FLAGS_backend);
// Run initialization network.
caffe2::NetDef init_net_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net_def));
setOperatorEngine(&init_net_def, FLAGS_backend);
CAFFE_ENFORCE(workspace->RunNetOnce(init_net_def));
// Run main network.
caffe2::NetDef net_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_net, &net_def));
setOperatorEngine(&net_def, FLAGS_backend);
map<string, caffe2::TensorProtos> tensor_protos_map;
int num_blobs;
vector<string> images = caffe2::split(';', FLAGS_input_image_files);
for (int i = 0; i < images.size(); ++i) {
vector<string> mapping = caffe2::split(',', images[i]);
caffe2::TensorProtos proto_images = caffe2::convertImages(mapping[1]);
workspace->CreateBlob(mapping[0]);
tensor_protos_map.insert(std::make_pair(mapping[0], proto_images));
num_blobs = proto_images.protos_size();
}
vector<string> values = caffe2::split(';', FLAGS_input_text_files);
for (int i = 0; i < values.size(); ++i) {
vector<string> mapping = caffe2::split(',', values[i]);
caffe2::TensorProtos proto_values = caffe2::convertValues(mapping[1]);
workspace->CreateBlob(mapping[0]);
tensor_protos_map.insert(std::make_pair(mapping[0], proto_values));
num_blobs = proto_values.protos_size();
}
runNetwork(
workspace,
net_def,
tensor_protos_map,
FLAGS_wipe_cache,
FLAGS_run_individual,
run_on_gpu,
FLAGS_text_output,
FLAGS_warmup,
FLAGS_iter,
num_blobs,
FLAGS_sleep_before_run,
FLAGS_sleep_between_iteration,
FLAGS_sleep_between_net_and_operator,
FLAGS_output,
FLAGS_output_folder);
return 0;
}
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
benchmark(
argc,
argv,
FLAGS_backend,
FLAGS_init_net,
FLAGS_input_dims,
FLAGS_iter,
FLAGS_net,
FLAGS_output,
FLAGS_output_folder,
FLAGS_run_individual,
FLAGS_sleep_before_run,
FLAGS_sleep_between_iteration,
FLAGS_sleep_between_net_and_operator,
FLAGS_text_output,
FLAGS_warmup,
FLAGS_wipe_cache);
return 0;
}

90
binaries/convert_caffe_image_db.cc
View File

@ -1,90 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/proto/caffe2_legacy.pb.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(input_db, "", "The input db.");
C10_DEFINE_string(input_db_type, "", "The input db type.");
C10_DEFINE_string(output_db, "", "The output db.");
C10_DEFINE_string(output_db_type, "", "The output db type.");
C10_DEFINE_int(batch_size, 1000, "The write batch size.");
using caffe2::db::Cursor;
using caffe2::db::DB;
using caffe2::db::Transaction;
using caffe2::CaffeDatum;
using caffe2::TensorProto;
using caffe2::TensorProtos;
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
std::unique_ptr<DB> out_db(caffe2::db::CreateDB(
FLAGS_output_db_type, FLAGS_output_db, caffe2::db::NEW));
std::unique_ptr<Cursor> cursor(in_db->NewCursor());
std::unique_ptr<Transaction> transaction(out_db->NewTransaction());
int count = 0;
for (; cursor->Valid(); cursor->Next()) {
CaffeDatum datum;
CAFFE_ENFORCE(datum.ParseFromString(cursor->value()));
TensorProtos protos;
TensorProto* data = protos.add_protos();
TensorProto* label = protos.add_protos();
label->set_data_type(TensorProto::INT32);
label->add_dims(1);
label->add_int32_data(datum.label());
if (datum.encoded()) {
// This is an encoded image. we will copy over the data directly.
data->set_data_type(TensorProto::STRING);
data->add_dims(1);
data->add_string_data(datum.data());
} else {
// float data not supported right now.
CAFFE_ENFORCE_EQ(datum.float_data_size(), 0);
std::vector<char> buffer_vec(datum.data().size());
char* buffer = buffer_vec.data();
// swap order from CHW to HWC
int channels = datum.channels();
int size = datum.height() * datum.width();
CAFFE_ENFORCE_EQ(datum.data().size(), channels * size);
for (int c = 0; c < channels; ++c) {
char* dst = buffer + c;
const char* src = datum.data().c_str() + c * size;
for (int n = 0; n < size; ++n) {
dst[n*channels] = src[n];
}
}
data->set_data_type(TensorProto::BYTE);
data->add_dims(datum.height());
data->add_dims(datum.width());
data->add_dims(datum.channels());
data->set_byte_data(buffer, datum.data().size());
}
transaction->Put(cursor->key(), SerializeAsString_EnforceCheck(protos));
if (++count % FLAGS_batch_size == 0) {
transaction->Commit();
LOG(INFO) << "Converted " << count << " items so far.";
}
}
LOG(INFO) << "A total of " << count << " items processed.";
return 0;
}

51
binaries/convert_db.cc
View File

@ -1,51 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(input_db, "", "The input db.");
C10_DEFINE_string(input_db_type, "", "The input db type.");
C10_DEFINE_string(output_db, "", "The output db.");
C10_DEFINE_string(output_db_type, "", "The output db type.");
C10_DEFINE_int(batch_size, 1000, "The write batch size.");
using caffe2::db::Cursor;
using caffe2::db::DB;
using caffe2::db::Transaction;
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
std::unique_ptr<DB> out_db(caffe2::db::CreateDB(
FLAGS_output_db_type, FLAGS_output_db, caffe2::db::NEW));
std::unique_ptr<Cursor> cursor(in_db->NewCursor());
std::unique_ptr<Transaction> transaction(out_db->NewTransaction());
int count = 0;
for (; cursor->Valid(); cursor->Next()) {
transaction->Put(cursor->key(), cursor->value());
if (++count % FLAGS_batch_size == 0) {
transaction->Commit();
LOG(INFO) << "Converted " << count << " items so far.";
}
}
LOG(INFO) << "A total of " << count << " items processed.";
return 0;
}

155
binaries/convert_encoded_to_raw_leveldb.cc
View File

@ -1,155 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This script converts an image dataset to leveldb.
//
// FLAGS_input_folder is the root folder that holds all the images, and
// FLAGS_list_file should be a list of files as well as their labels, in
// the format as
// subfolder1/file1.JPEG 7
// ....
#include <opencv2/opencv.hpp>
#include <fstream> // NOLINT(readability/streams)
#include <memory>
#include <random>
#include <string>
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/core/logging.h"
#include "leveldb/db.h"
#include "leveldb/write_batch.h"
C10_DEFINE_string(input_db_name, "", "The input image file name.");
C10_DEFINE_string(output_db_name, "", "The output training leveldb name.");
C10_DEFINE_bool(color, true, "If set, load images in color.");
C10_DEFINE_int(
scale,
256,
"If FLAGS_raw is set, scale all the images' shorter edge to the given "
"value.");
C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
namespace caffe2 {
void ConvertToRawDataset(
const string& input_db_name, const string& output_db_name) {
// input leveldb
std::unique_ptr<leveldb::DB> input_db;
LOG(INFO) << "Opening input leveldb " << input_db_name;
{
leveldb::Options options;
options.create_if_missing = false;
leveldb::DB* db_temp;
leveldb::Status status = leveldb::DB::Open(
options, input_db_name, &db_temp);
CAFFE_ENFORCE(status.ok(), "Failed to open leveldb ", input_db_name, ".");
input_db.reset(db_temp);
}
// output leveldb
std::unique_ptr<leveldb::DB> output_db;
std::unique_ptr<leveldb::WriteBatch> batch;
LOG(INFO) << "Opening leveldb " << output_db_name;
{
leveldb::Options options;
options.error_if_exists = true;
options.create_if_missing = true;
options.write_buffer_size = 268435456;
leveldb::DB* db_temp;
leveldb::Status status = leveldb::DB::Open(
options, output_db_name, &db_temp);
CAFFE_ENFORCE(
status.ok(),
"Failed to open leveldb ",
output_db_name,
". Is it already existing?");
output_db.reset(db_temp);
}
batch.reset(new leveldb::WriteBatch());
TensorProtos input_protos;
TensorProtos output_protos;
TensorProto* data = output_protos.add_protos();
TensorProto* label = output_protos.add_protos();
data->set_data_type(TensorProto::BYTE);
data->add_dims(0);
data->add_dims(0);
if (FLAGS_color) {
data->add_dims(3);
}
string value;
unique_ptr<leveldb::Iterator> iter;
iter.reset(input_db->NewIterator(leveldb::ReadOptions()));
iter->SeekToFirst();
int count = 0;
for (; iter->Valid(); iter->Next()) {
CAFFE_ENFORCE(input_protos.ParseFromString(iter->value().ToString()));
label->CopyFrom(input_protos.protos(1));
const string& encoded_image = input_protos.protos(0).string_data(0);
int encoded_size = encoded_image.size();
cv::Mat img = cv::imdecode(
cv::Mat(
1, &encoded_size, CV_8UC1, const_cast<char*>(encoded_image.data())),
FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
cv::Mat resized_img;
int scaled_width, scaled_height;
if (FLAGS_warp) {
scaled_width = FLAGS_scale;
scaled_height = FLAGS_scale;
} else if (img.rows > img.cols) {
scaled_width = FLAGS_scale;
scaled_height = static_cast<float>(img.rows) * FLAGS_scale / img.cols;
} else {
scaled_height = FLAGS_scale;
scaled_width = static_cast<float>(img.cols) * FLAGS_scale / img.rows;
}
cv::resize(img, resized_img, cv::Size(scaled_width, scaled_height), 0, 0,
cv::INTER_LINEAR);
data->set_dims(0, scaled_height);
data->set_dims(1, scaled_width);
DCHECK(resized_img.isContinuous());
data->set_byte_data(
resized_img.ptr(),
scaled_height * scaled_width * (FLAGS_color ? 3 : 1));
output_protos.SerializeToString(&value);
// Put in db
batch->Put(iter->key(), value);
if (++count % 1000 == 0) {
output_db->Write(leveldb::WriteOptions(), batch.get());
batch.reset(new leveldb::WriteBatch());
LOG(INFO) << "Processed " << count << " files.";
}
}
// write the last batch
if (count % 1000 != 0) {
output_db->Write(leveldb::WriteOptions(), batch.get());
}
LOG(INFO) << "Processed a total of " << count << " files.";
}
} // namespace caffe2
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::ConvertToRawDataset(FLAGS_input_db_name, FLAGS_output_db_name);
return 0;
}

450
binaries/convert_image_to_tensor.cc
View File

@ -1,450 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <opencv2/opencv.hpp>
#include <cmath>
#include <fstream>
#include "caffe2/core/common.h"
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/timer.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/utils/proto_utils.h"
#include "caffe2/utils/string_utils.h"
C10_DEFINE_int(
batch_size,
-1,
"Specify the batch size of the input. The number of items in the "
"input needs to be multiples of the batch size. If the batch size "
"is less than 0, all inputs are in one batch.")
C10_DEFINE_bool(color, true, "If set, load images in color.");
C10_DEFINE_string(
crop,
"-1,-1",
"The center cropped hight and width. If the value is less than zero, "
"it is not cropped.");
C10_DEFINE_string(input_images, "", "Comma separated images");
C10_DEFINE_string(input_image_file, "", "The file containing imput images");
C10_DEFINE_string(input_text_file, "", "the text file to be written to blobs");
C10_DEFINE_string(
output_tensor,
"",
"The output tensor file in NCHW for input images");
C10_DEFINE_string(
output_text_tensor,
"",
"The output tensor file for the text input specified in input_text_file");
C10_DEFINE_string(
preprocess,
"",
"Options to specify the preprocess routines. The available options are "
"subtract128, normalize, mean, std, bgrtorgb. If multiple steps are provided, they "
"are separated by comma (,) in sequence.");
C10_DEFINE_string(
report_time,
"",
"Report the conversion stage time to screen. "
"The format of the string is <type>|<identifier>. "
"The valid type is 'json'. "
"The valid identifier is nothing or an identifier that prefix every line");
C10_DEFINE_string(
scale,
"-1,-1",
"Scale the images to be within the min,max box. The shorter edge is "
"min pixels. But if the other edge is more than the max pixels, the "
"other edge and scaled to max pixels (and the shorter edge can be less "
"than the min pixels");
C10_DEFINE_bool(text_output, false, "Write the output in text format.");
C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
namespace caffe2 {
void reportTime(
std::string type,
double ts,
std::string metric,
std::string unit) {
if (FLAGS_report_time == "") {
return;
}
vector<string> s = caffe2::split('|', FLAGS_report_time);
assert(s[0] == "json");
std::string identifier = "";
if (s.size() > 1) {
identifier = s[1];
}
std::cout << identifier << "{\"type\": \"" << type << "\", \"value\": " << ts
<< ", \"metric\": \"" << metric << "\", \"unit\": \"" << unit
<< "\"}" << std::endl;
}
void splitSizes(const std::string& arg, int* ptr0, int* ptr1) {
vector<string> sizes = caffe2::split(',', arg);
if (sizes.size() == 2) {
*ptr0 = std::stoi(sizes[0]);
*ptr1 = std::stoi(sizes[1]);
} else if (sizes.size() == 1) {
*ptr0 = std::stoi(sizes[0]);
*ptr1 = std::stoi(sizes[0]);
} else {
assert(false);
}
}
cv::Mat resizeImage(cv::Mat& img) {
int min_size, max_size;
splitSizes(FLAGS_scale, &min_size, &max_size);
if ((min_size <= 0) && (max_size <= 0)) {
return img;
}
if (max_size < 0) {
max_size = INT_MAX;
}
assert(min_size <= max_size);
int im_min_size = img.rows > img.cols ? img.cols : img.rows;
int im_max_size = img.rows > img.cols ? img.rows : img.cols;
double im_scale = 1.0 * min_size / im_min_size;
if (im_scale * im_max_size > max_size) {
im_scale = 1.0 * max_size / im_max_size;
}
int scaled_width = int(round(img.cols * im_scale));
int scaled_height = int(round(img.rows * im_scale));
assert((scaled_width <= max_size) && (scaled_height <= max_size));
if ((scaled_width < min_size) || (scaled_height < min_size)) {
assert((scaled_width == max_size) || (scaled_height == max_size));
} else {
assert((scaled_width == min_size) || (scaled_height == min_size));
}
cv::Mat resized_img;
cv::resize(
img,
resized_img,
cv::Size(),
im_scale,
im_scale,
cv::INTER_LINEAR);
return resized_img;
}
cv::Mat cropToRec(cv::Mat& img, int* height_ptr, int* width_ptr) {
int height = *height_ptr;
int width = *width_ptr;
if ((height > 0) && (width > 0) &&
((img.rows != height) || (img.cols != width))) {
cv::Mat cropped_img, cimg;
cv::Rect roi;
roi.x = int((img.cols - width) / 2);
roi.y = int((img.rows - height) / 2);
roi.x = roi.x < 0 ? 0 : roi.x;
roi.y = roi.y < 0 ? 0 : roi.y;
width = width > img.cols ? img.cols : width;
height = height > img.rows ? img.rows : height;
roi.width = width;
roi.height = height;
assert(
0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= img.cols &&
0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= img.rows);
cropped_img = img(roi);
// Make the image in continuous space in memory
cimg = cropped_img.clone();
*height_ptr = height;
*width_ptr = width;
return cimg;
} else {
return img;
}
}
std::vector<float> convertToVector(cv::Mat& img) {
std::vector<float> normalize(3, 1);
std::vector<float> mean(3, 0);
std::vector<float> std(3, 1);
bool bgrtorgb = false;
int size = img.cols * img.rows;
vector<string> steps = caffe2::split(',', FLAGS_preprocess);
for (int i = 0; i < steps.size(); i++) {
auto step = steps[i];
if (step == "subtract128") {
mean = {128, 128, 128};
std = {1, 1, 1};
normalize = {1, 1, 1};
} else if (step == "normalize") {
normalize = {255, 255, 255};
} else if (step == "mean") {
mean = {0.406f, 0.456f, 0.485f};
} else if (step == "std") {
std = {0.225f, 0.224f, 0.229f};
} else if (step == "bgrtorgb") {
bgrtorgb = true;
} else {
CAFFE_ENFORCE(
false,
"Unsupported preprocess step. The supported steps are: subtract128, "
"normalize,mean, std, swaprb.");
}
}
int C = FLAGS_color ? 3 : 1;
int total_size = C * size;
std::vector<float> values(total_size);
if (C == 1) {
cv::MatIterator_<float> it, end;
int idx = 0;
for (it = img.begin<float>(), end = img.end<float>(); it != end; ++it) {
values[idx++] = (*it / normalize[0] - mean[0]) / std[0];
}
} else {
int i = 0;
cv::MatIterator_<cv::Vec3f> it, end;
int b = bgrtorgb ? 2 : 0;
int g = 1;
int r = bgrtorgb ? 0 : 2;
for (it = img.begin<cv::Vec3f>(), end = img.end<cv::Vec3f>(); it != end;
++it, i++) {
values[i] = (((*it)[b] / normalize[0] - mean[0]) / std[0]);
int offset = size + i;
values[offset] = (((*it)[g] / normalize[1] - mean[1]) / std[1]);
offset = size + offset;
values[offset] = (((*it)[r] / normalize[2] - mean[2]) / std[2]);
}
}
return values;
}
std::vector<float> convertOneImage(
std::string& filename,
int* height_ptr,
int* width_ptr) {
assert(filename[0] != '~');
std::cout << "Converting " << filename << std::endl;
// Load image
cv::Mat img_uint8 = cv::imread(
#if CV_MAJOR_VERSION <= 3
filename, FLAGS_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);
#else
filename, FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
#endif
caffe2::Timer timer;
timer.Start();
cv::Mat img;
// Convert image to floating point values
img_uint8.convertTo(img, CV_32F);
// Resize image
cv::Mat resized_img = resizeImage(img);
int height, width;
splitSizes(FLAGS_crop, &height, &width);
if ((height <= 0) || (width <= 0)) {
height = resized_img.rows;
width = resized_img.cols;
}
cv::Mat crop = cropToRec(resized_img, &height, &width);
// Assert we don't have to deal with alignment
DCHECK(crop.isContinuous());
assert(crop.rows == height);
assert(crop.cols == width);
std::vector<float> one_image_values = convertToVector(crop);
*height_ptr = height;
*width_ptr = width;
double ts = timer.MicroSeconds();
reportTime("image_preprocess", ts, "convert", "us");
return one_image_values;
}
int getBatchSize(int num_items) {
int batch_size = FLAGS_batch_size;
if (batch_size < 0) {
batch_size = num_items;
} else {
assert(num_items % batch_size == 0);
}
return batch_size;
}
void writeValues(
std::vector<std::vector<std::vector<float>>>& values,
std::vector<std::vector<int>>& dims,
std::string output_file) {
caffe2::Timer timer;
timer.Start();
assert(dims.size() == values.size());
int num_batches = dims.size();
TensorProtos protos;
for (int k = 0; k < num_batches; k++) {
TensorProto* data;
data = protos.add_protos();
data->set_data_type(TensorProto::FLOAT);
auto one_dim = dims[k];
for (int dim : one_dim) {
data->add_dims(dim);
}
int batch_size = one_dim[0];
long long int entry_size = 1;
for (int i = 1; i < one_dim.size(); i++) {
entry_size *= one_dim[i];
}
// Not optimized
for (int i = 0; i < batch_size; i++) {
assert(values[k][i].size() == entry_size);
for (int j = 0; j < values[k][i].size(); j++) {
data->add_float_data(values[k][i][j]);
}
}
}
double ts = timer.MicroSeconds();
reportTime("preprocess", ts, "data_pack", "us");
if (FLAGS_text_output) {
caffe2::WriteProtoToTextFile(protos, output_file);
} else {
caffe2::WriteProtoToBinaryFile(protos, output_file);
}
}
void convertImages() {
vector<string> file_names;
if (FLAGS_input_images != "") {
file_names = caffe2::split(',', FLAGS_input_images);
} else if (FLAGS_input_image_file != "") {
std::ifstream infile(FLAGS_input_image_file);
std::string line;
while (std::getline(infile, line)) {
vector<string> file_name = caffe2::split(',', line);
string name;
if (file_name.size() == 3) {
name = file_name[2];
} else {
name = line;
}
file_names.push_back(name);
}
} else {
return;
}
int batch_size = getBatchSize(file_names.size());
int num_batches = file_names.size() / batch_size;
assert(file_names.size() == batch_size * num_batches);
std::vector<std::vector<std::vector<float>>> values;
std::vector<std::vector<int>> dims;
int C = FLAGS_color ? 3 : 1;
for (int k = 0; k < num_batches; k++) {
std::vector<std::vector<float>> one_value;
int height = -1;
int width = -1;
for (int i = 0; i < batch_size; i++) {
int idx = k * batch_size + i;
int one_height, one_width;
std::vector<float> one_image_values =
convertOneImage(file_names[idx], &one_height, &one_width);
if (height < 0 && width < 0) {
height = one_height;
width = one_width;
} else {
assert(height == one_height);
assert(width == one_width);
}
one_value.push_back(one_image_values);
}
vector<int> one_dim = {batch_size, C, height, width};
dims.push_back(one_dim);
values.push_back(one_value);
}
writeValues(values, dims, FLAGS_output_tensor);
}
template <class TYPE>
vector<TYPE> splitString(std::string& line) {
vector<string> vector_str = caffe2::split(',', line);
vector<TYPE> vector_int;
for (string str : vector_str) {
vector_int.push_back((TYPE)std::stod(str));
}
return vector_int;
}
/* Convert the values in a json file to blobs
The format of the json file should be:
<number of items>, <dim2>.... (dimensions of items)
<entry>, <entry>, <entry>... (all entries in one item)
<entry>, <entry>, <entry>...
....
*/
void convertValues() {
if (FLAGS_input_text_file == "") {
return;
}
std::ifstream infile(FLAGS_input_text_file);
std::string line;
std::getline(infile, line);
vector<int> file_dims = splitString <int>(line);
assert(file_dims.size() >= 2);
int num_items = file_dims[0];
int batch_size = getBatchSize(num_items);
int num_batches = num_items / batch_size;
assert(num_items == batch_size * num_batches);
vector<string> lines;
while (std::getline(infile, line)) {
lines.push_back(line);
}
assert(lines.size() == num_items);
std::vector<std::vector<std::vector<float>>> values;
std::vector<std::vector<int>> dims;
for (int i = 0; i < num_batches; i++) {
std::vector<std::vector<float>> one_value;
int num = -1;
for (int j = 0; j < batch_size; j++) {
int idx = i * batch_size + j;
std::string line = lines[idx];
vector<float> item = splitString<float>(line);
if (num < 0) {
num = item.size();
} else {
assert(num == item.size());
}
one_value.push_back(item);
}
vector<int> batch_dims = file_dims;
batch_dims[0] = batch_size;
dims.push_back(batch_dims);
values.push_back(one_value);
}
writeValues(values, dims, FLAGS_output_text_tensor);
}
} // namespace caffe2
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::convertImages();
caffe2::convertValues();
return 0;
}

104
binaries/db_throughput.cc
View File

@ -1,104 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstdio>
#include <thread>
#include <vector>
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/core/timer.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(input_db, "", "The input db.");
C10_DEFINE_string(input_db_type, "", "The input db type.");
C10_DEFINE_int(report_interval, 1000, "The report interval.");
C10_DEFINE_int(repeat, 10, "The number to repeat the throughput test.");
C10_DEFINE_bool(use_reader, false, "If true, use the reader interface.");
C10_DEFINE_int(
num_read_threads,
1,
"The number of concurrent reading threads.");
using caffe2::db::Cursor;
using caffe2::db::DB;
using caffe2::db::DBReader;
using caffe2::string;
void TestThroughputWithDB() {
std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
std::unique_ptr<Cursor> cursor(in_db->NewCursor());
for (int iter_id = 0; iter_id < FLAGS_repeat; ++iter_id) {
caffe2::Timer timer;
for (int i = 0; i < FLAGS_report_interval; ++i) {
string key = cursor->key();
string value = cursor->value();
//VLOG(1) << "Key " << key;
cursor->Next();
if (!cursor->Valid()) {
cursor->SeekToFirst();
}
}
double elapsed_seconds = timer.Seconds();
printf(
"Iteration %03d, took %4.5f seconds, throughput %f items/sec.\n",
iter_id,
elapsed_seconds,
FLAGS_report_interval / elapsed_seconds);
}
}
void TestThroughputWithReaderWorker(const DBReader* reader, int thread_id) {
string key, value;
for (int iter_id = 0; iter_id < FLAGS_repeat; ++iter_id) {
caffe2::Timer timer;
for (int i = 0; i < FLAGS_report_interval; ++i) {
reader->Read(&key, &value);
}
double elapsed_seconds = timer.Seconds();
printf(
"Thread %03d iteration %03d, took %4.5f seconds, "
"throughput %f items/sec.\n",
thread_id,
iter_id,
elapsed_seconds,
FLAGS_report_interval / elapsed_seconds);
}
}
void TestThroughputWithReader() {
caffe2::db::DBReader reader(FLAGS_input_db_type, FLAGS_input_db);
std::vector<std::unique_ptr<std::thread>> reading_threads(
FLAGS_num_read_threads);
for (int i = 0; i < reading_threads.size(); ++i) {
reading_threads[i].reset(new std::thread(
TestThroughputWithReaderWorker, &reader, i));
}
for (int i = 0; i < reading_threads.size(); ++i) {
reading_threads[i]->join();
}
}
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
if (FLAGS_use_reader) {
TestThroughputWithReader();
} else {
TestThroughputWithDB();
}
return 0;
}

166
binaries/intra_inter_benchmark.cc
View File

@ -1,166 +0,0 @@
#include "ATen/ATen.h"
#include "ATen/Parallel.h"
#include "c10/util/Flags.h"
#include "caffe2/core/init.h"
#include <chrono>
#include <condition_variable>
#include <ctime>
#include <iostream>
#include <mutex>
#include <thread>
C10_DEFINE_int(iter_pow, 10, "Number of tasks, 2^N");
C10_DEFINE_int(sub_iter, 1024, "Number of subtasks");
C10_DEFINE_int(warmup_iter_pow, 3, "Number of warmup tasks, 2^N");
C10_DEFINE_int(inter_op_threads, 0, "Number of inter-op threads");
C10_DEFINE_int(intra_op_threads, 0, "Number of intra-op threads");
C10_DEFINE_int(tensor_dim, 50, "Tensor dim");
C10_DEFINE_int(benchmark_iter, 10, "Number of times to run benchmark")
C10_DEFINE_bool(extra_stats, false,
"Collect extra stats; warning: skews results");
C10_DEFINE_string(task_type, "add", "Tensor operation: add or mm");
namespace {
std::atomic<int> counter{0};
int overall_tasks = 0;
std::condition_variable cv;
std::mutex tasks_mutex;
bool run_mm = false;
std::mutex stats_mutex;
std::unordered_set<std::thread::id> tids;
}
void wait() {
std::unique_lock<std::mutex> lk(tasks_mutex);
while (counter < overall_tasks) {
cv.wait(lk);
}
}
void _launch_tasks_tree(
int level, int end_level, at::Tensor& left, at::Tensor& right) {
if (level == end_level) {
at::parallel_for(0, FLAGS_sub_iter, 1,
[&left, &right](int64_t begin, int64_t end) {
if (FLAGS_extra_stats) {
std::unique_lock<std::mutex> lk(stats_mutex);
tids.insert(std::this_thread::get_id());
}
for (auto k = begin; k < end; ++k) {
if (run_mm) {
left.mm(right);
} else {
left.add(right);
}
auto cur_ctr = ++counter;
if (cur_ctr == overall_tasks) {
std::unique_lock<std::mutex> lk(tasks_mutex);
cv.notify_one();
}
}
});
} else {
at::launch([&left, &right, level, end_level]() {
_launch_tasks_tree(level + 1, end_level, left, right);
});
at::launch([&left, &right, level, end_level]() {
_launch_tasks_tree(level + 1, end_level, left, right);
});
}
};
void launch_tasks_and_wait(at::Tensor& left, at::Tensor& right, int iter_pow) {
overall_tasks = pow(2, iter_pow) * FLAGS_sub_iter;
counter = 0;
_launch_tasks_tree(0, iter_pow, left, right);
wait();
}
void reset_extra_stats() {
tids.clear();
}
void print_extra_stats() {
std::cout << "# threads: " << tids.size() << std::endl;
}
void print_runtime_stats(const std::vector<float>& runtimes) {
TORCH_INTERNAL_ASSERT(!runtimes.empty());
float sum = 0.0;
float sqr_sum = 0.0;
size_t N = runtimes.size();
for (size_t idx = 0; idx < N; ++idx) {
sum += runtimes[idx];
sqr_sum += runtimes[idx] * runtimes[idx];
}
float mean = sum / N;
float sd = std::sqrt(sqr_sum / N - mean * mean);
std::cout << "N = " << N << ", mean = " << mean << ", sd = " << sd
<< std::endl;
}
int main(int argc, char** argv) {
if (!c10::ParseCommandLineFlags(&argc, &argv)) {
std::cout << "Failed to parse command line flags" << std::endl;
return -1;
}
caffe2::unsafeRunCaffe2InitFunction("registerThreadPools");
at::init_num_threads();
if (FLAGS_inter_op_threads > 0) {
at::set_num_interop_threads(FLAGS_inter_op_threads);
}
if (FLAGS_intra_op_threads > 0) {
at::set_num_threads(FLAGS_intra_op_threads);
}
TORCH_CHECK(FLAGS_task_type == "add" || FLAGS_task_type == "mm");
run_mm = FLAGS_task_type == "mm";
auto left = at::ones({FLAGS_tensor_dim, FLAGS_tensor_dim}, at::kFloat);
auto right = at::ones({FLAGS_tensor_dim, FLAGS_tensor_dim}, at::kFloat);
std::cout << "Launching " << pow(2, FLAGS_warmup_iter_pow)
<< " warmup tasks" << std::endl;
typedef std::chrono::high_resolution_clock clock;
typedef std::chrono::milliseconds ms;
std::chrono::time_point<clock> start_time = clock::now();
launch_tasks_and_wait(left, right, FLAGS_warmup_iter_pow);
auto duration = static_cast<float>(
std::chrono::duration_cast<ms>(clock::now() - start_time).count());
std::cout << "Warmup time: " << duration << " ms." << std::endl;
std::cout << "Launching " << pow(2, FLAGS_iter_pow) << " tasks with "
<< FLAGS_sub_iter << " subtasks each, using "
<< at::get_num_interop_threads() << " inter-op threads and "
<< at::get_num_threads() << " intra-op threads, "
<< "tensor dim: " << FLAGS_tensor_dim
<< ", task type: " << FLAGS_task_type << std::endl;
std::vector<float> runtimes;
for (auto bench_iter = 0; bench_iter < FLAGS_benchmark_iter; ++bench_iter) {
reset_extra_stats();
start_time = clock::now();
launch_tasks_and_wait(left, right, FLAGS_iter_pow);
duration = static_cast<float>(
std::chrono::duration_cast<ms>(clock::now() - start_time).count());
runtimes.push_back(duration);
if (FLAGS_extra_stats) {
print_extra_stats();
}
std::cout << "Runtime: " << duration << " ms." << std::endl;
}
print_runtime_stats(runtimes);
return 0;
}

144
binaries/make_cifar_db.cc
View File

@ -1,144 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//
// This script converts the CIFAR dataset to the leveldb format used
// by caffe to perform classification.
// Usage:
// convert_cifar_data input_folder output_db_file
// The CIFAR dataset could be downloaded at
// http://www.cs.toronto.edu/~kriz/cifar.html
#include <array>
#include <fstream> // NOLINT(readability/streams)
#include <sstream>
#include <string>
#include "caffe2/core/common.h"
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(input_folder, "", "The input folder name.");
C10_DEFINE_string(output_train_db_name, "", "The output training db name.");
C10_DEFINE_string(output_test_db_name, "", "The output testing db name.");
C10_DEFINE_string(db, "leveldb", "The db type.");
C10_DEFINE_bool(
is_cifar100,
false,
"If set, convert cifar100. Otherwise do cifar10.");
namespace caffe2 {
using std::stringstream;
const int kCIFARSize = 32;
const int kCIFARImageNBytes = kCIFARSize * kCIFARSize * 3;
const int kCIFAR10BatchSize = 10000;
const int kCIFAR10TestDataSize = 10000;
const int kCIFAR10TrainBatches = 5;
const int kCIFAR100TrainDataSize = 50000;
const int kCIFAR100TestDataSize = 10000;
void ReadImage(std::ifstream* file, int* label, char* buffer) {
char label_char;
if (FLAGS_is_cifar100) {
// Skip the coarse label.
file->read(&label_char, 1);
}
file->read(&label_char, 1);
*label = label_char;
// Yes, there are better ways to do it, like in-place swap... but I am too
// lazy so let's just write it in a memory-wasteful way.
std::array<char, kCIFARImageNBytes> channel_first_storage;
file->read(channel_first_storage.data(), kCIFARImageNBytes);
for (int c = 0; c < 3; ++c) {
for (int i = 0; i < kCIFARSize * kCIFARSize; ++i) {
buffer[i * 3 + c] =
channel_first_storage[c * kCIFARSize * kCIFARSize + i];
}
}
return;
}
void WriteToDB(const string& filename, const int num_items,
const int& offset, db::DB* db) {
TensorProtos protos;
TensorProto* data = protos.add_protos();
TensorProto* label = protos.add_protos();
data->set_data_type(TensorProto::BYTE);
data->add_dims(kCIFARSize);
data->add_dims(kCIFARSize);
data->add_dims(3);
label->set_data_type(TensorProto::INT32);
label->add_dims(1);
label->add_int32_data(0);
LOG(INFO) << "Converting file " << filename;
std::ifstream data_file(filename.c_str(),
std::ios::in | std::ios::binary);
CAFFE_ENFORCE(data_file, "Unable to open file ", filename);
char str_buffer[kCIFARImageNBytes];
int label_value;
std::unique_ptr<db::Transaction> transaction(db->NewTransaction());
for (int itemid = 0; itemid < num_items; ++itemid) {
ReadImage(&data_file, &label_value, str_buffer);
data->set_byte_data(str_buffer, kCIFARImageNBytes);
label->set_int32_data(0, label_value);
snprintf(str_buffer, kCIFARImageNBytes, "%05d",
offset + itemid);
transaction->Put(string(str_buffer), protos.SerializeAsString());
}
}
void ConvertCIFAR() {
std::unique_ptr<db::DB> train_db(
db::CreateDB(FLAGS_db, FLAGS_output_train_db_name, db::NEW));
std::unique_ptr<db::DB> test_db(
db::CreateDB(FLAGS_db, FLAGS_output_test_db_name, db::NEW));
if (!FLAGS_is_cifar100) {
// This is cifar 10.
for (int fileid = 0; fileid < kCIFAR10TrainBatches; ++fileid) {
stringstream train_file;
train_file << FLAGS_input_folder << "/data_batch_" << fileid + 1
<< ".bin";
WriteToDB(train_file.str(), kCIFAR10BatchSize,
fileid * kCIFAR10BatchSize, train_db.get());
}
stringstream test_file;
test_file << FLAGS_input_folder << "/test_batch.bin";
WriteToDB(test_file.str(), kCIFAR10TestDataSize, 0, test_db.get());
} else {
// This is cifar 100.
stringstream train_file;
train_file << FLAGS_input_folder << "/train.bin";
WriteToDB(train_file.str(), kCIFAR100TrainDataSize, 0, train_db.get());
stringstream test_file;
test_file << FLAGS_input_folder << "/test.bin";
WriteToDB(test_file.str(), kCIFAR100TestDataSize, 0, test_db.get());
}
}
} // namespace caffe2
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::ConvertCIFAR();
return 0;
}

279
binaries/make_image_db.cc
View File

@ -1,279 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This script converts an image dataset to a database.
//
// FLAGS_input_folder is the root folder that holds all the images
//
// FLAGS_list_file is the path to a file containing a list of files
// and their labels, as follows:
//
// subfolder1/file1.JPEG 7
// subfolder1/file2.JPEG 7
// subfolder2/file1.JPEG 8
// ...
//
#include <opencv2/opencv.hpp>
#include <algorithm>
#include <fstream>
#include <queue>
#include <random>
#include <string>
#include <thread>
#include "caffe2/core/common.h"
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/core/logging.h"
C10_DEFINE_bool(
shuffle,
false,
"Randomly shuffle the order of images and their labels");
C10_DEFINE_string(input_folder, "", "The input image file name.");
C10_DEFINE_string(
list_file,
"",
"The text file containing the list of images.");
C10_DEFINE_string(output_db_name, "", "The output training leveldb name.");
C10_DEFINE_string(db, "leveldb", "The db type.");
C10_DEFINE_bool(
raw,
false,
"If set, we pre-read the images and store the raw buffer.");
C10_DEFINE_bool(color, true, "If set, load images in color.");
C10_DEFINE_int(
scale,
256,
"If FLAGS_raw is set, scale the shorter edge to the given value.");
C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
C10_DEFINE_int(
num_threads,
-1,
"Number of image parsing and conversion threads.");
namespace caffe2 {
class Converter {
public:
explicit Converter() {
data_ = protos_.add_protos();
label_ = protos_.add_protos();
if (FLAGS_raw) {
data_->set_data_type(TensorProto::BYTE);
data_->add_dims(0);
data_->add_dims(0);
if (FLAGS_color) {
data_->add_dims(3);
}
} else {
data_->set_data_type(TensorProto::STRING);
data_->add_dims(1);
data_->add_string_data("");
}
label_->set_data_type(TensorProto::INT32);
label_->add_dims(1);
label_->add_int32_data(0);
}
~Converter() {
if (thread_.joinable()) {
thread_.join();
}
}
void queue(const std::pair<std::string, int>& pair) {
in_.push(pair);
}
void start() {
thread_ = std::thread(&Converter::run, this);
}
std::string get() {
std::unique_lock<std::mutex> lock(mutex_);
while (out_.empty()) {
cv_.wait(lock);
}
auto value = out_.front();
out_.pop();
cv_.notify_one();
return value;
}
void run() {
const auto& input_folder = FLAGS_input_folder;
std::unique_lock<std::mutex> lock(mutex_);
std::string value;
while (!in_.empty()) {
auto pair = in_.front();
in_.pop();
lock.unlock();
label_->set_int32_data(0, pair.second);
// Add raw file contents to DB if !raw
if (!FLAGS_raw) {
std::ifstream image_file_stream(input_folder + pair.first);
if (!image_file_stream) {
LOG(ERROR) << "Cannot open " << input_folder << pair.first
<< ". Skipping.";
} else {
data_->mutable_string_data(0)->assign(
std::istreambuf_iterator<char>(image_file_stream),
std::istreambuf_iterator<char>());
}
} else {
// Load image
cv::Mat img = cv::imread(
input_folder + pair.first,
FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
// Resize image
cv::Mat resized_img;
int scaled_width, scaled_height;
if (FLAGS_warp) {
scaled_width = FLAGS_scale;
scaled_height = FLAGS_scale;
} else if (img.rows > img.cols) {
scaled_width = FLAGS_scale;
scaled_height = static_cast<float>(img.rows) * FLAGS_scale / img.cols;
} else {
scaled_height = FLAGS_scale;
scaled_width = static_cast<float>(img.cols) * FLAGS_scale / img.rows;
}
cv::resize(
img,
resized_img,
cv::Size(scaled_width, scaled_height),
0,
0,
cv::INTER_LINEAR);
data_->set_dims(0, scaled_height);
data_->set_dims(1, scaled_width);
// Assert we don't have to deal with alignment
DCHECK(resized_img.isContinuous());
auto nbytes = resized_img.total() * resized_img.elemSize();
data_->set_byte_data(resized_img.ptr(), nbytes);
}
protos_.SerializeToString(&value);
// Add serialized proto to out queue or wait if it is not empty
lock.lock();
while (!out_.empty()) {
cv_.wait(lock);
}
out_.push(value);
cv_.notify_one();
}
}
protected:
TensorProtos protos_;
TensorProto* data_;
TensorProto* label_;
std::queue<std::pair<std::string, int>> in_;
std::queue<std::string> out_;
std::mutex mutex_;
std::condition_variable cv_;
std::thread thread_;
};
void ConvertImageDataset(
const string& input_folder,
const string& list_filename,
const string& output_db_name,
const bool /*shuffle*/) {
std::ifstream list_file(list_filename);
std::vector<std::pair<std::string, int> > lines;
std::string filename;
int file_label;
while (list_file >> filename >> file_label) {
lines.push_back(std::make_pair(filename, file_label));
}
if (FLAGS_shuffle) {
LOG(INFO) << "Shuffling data";
std::shuffle(lines.begin(), lines.end(), std::default_random_engine(1701));
}
auto num_threads = FLAGS_num_threads;
if (num_threads < 1) {
num_threads = std::thread::hardware_concurrency();
}
LOG(INFO) << "Processing " << lines.size() << " images...";
LOG(INFO) << "Opening DB " << output_db_name;
auto db = db::CreateDB(FLAGS_db, output_db_name, db::NEW);
auto transaction = db->NewTransaction();
LOG(INFO) << "Using " << num_threads << " processing threads...";
std::vector<Converter> converters(num_threads);
// Queue entries across converters
for (auto i = 0; i < lines.size(); i++) {
converters[i % converters.size()].queue(lines[i]);
}
// Start all converters
for (auto& converter : converters) {
converter.start();
}
constexpr auto key_max_length = 256;
char key_cstr[key_max_length];
int count = 0;
for (auto i = 0; i < lines.size(); i++) {
// Get serialized proto for this entry
auto value = converters[i % converters.size()].get();
// Synthesize key for this entry
auto key_len = snprintf(
key_cstr, sizeof(key_cstr), "%08d_%s", i, lines[i].first.c_str());
TORCH_DCHECK_LE(key_len, sizeof(key_cstr));
// Put in db
transaction->Put(string(key_cstr), std::move(value));
if (++count % 1000 == 0) {
// Commit the current writes.
transaction->Commit();
LOG(INFO) << "Processed " << count << " files.";
}
}
// Commit final transaction
transaction->Commit();
LOG(INFO) << "Processed " << count << " files.";
}
} // namespace caffe2
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::ConvertImageDataset(
FLAGS_input_folder, FLAGS_list_file, FLAGS_output_db_name, FLAGS_shuffle);
return 0;
}

144
binaries/make_mnist_db.cc
View File

@ -1,144 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This script converts the MNIST dataset to leveldb.
// The MNIST dataset could be downloaded at
// http://yann.lecun.com/exdb/mnist/
#include <fstream> // NOLINT(readability/streams)
#include <string>
#include "caffe2/core/common.h"
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(image_file, "", "The input image file name.");
C10_DEFINE_string(label_file, "", "The label file name.");
C10_DEFINE_string(output_file, "", "The output db name.");
C10_DEFINE_string(db, "leveldb", "The db type.");
C10_DEFINE_int(
data_limit,
-1,
"If set, only output this number of data points.");
C10_DEFINE_bool(
channel_first,
false,
"If set, write the data as channel-first (CHW order) as the old "
"Caffe does.");
namespace caffe2 {
uint32_t swap_endian(uint32_t val) {
val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
return (val << 16) | (val >> 16);
}
void convert_dataset(const char* image_filename, const char* label_filename,
const char* db_path, const int data_limit) {
// Open files
std::ifstream image_file(image_filename, std::ios::in | std::ios::binary);
std::ifstream label_file(label_filename, std::ios::in | std::ios::binary);
CAFFE_ENFORCE(image_file, "Unable to open file ", image_filename);
CAFFE_ENFORCE(label_file, "Unable to open file ", label_filename);
// Read the magic and the meta data
uint32_t magic;
uint32_t num_items;
uint32_t num_labels;
uint32_t rows;
uint32_t cols;
image_file.read(reinterpret_cast<char*>(&magic), 4);
magic = swap_endian(magic);
if (magic == 529205256) {
LOG(FATAL) <<
"It seems that you forgot to unzip the mnist dataset. You should "
"first unzip them using e.g. gunzip on Linux.";
}
CAFFE_ENFORCE_EQ(magic, 2051, "Incorrect image file magic.");
label_file.read(reinterpret_cast<char*>(&magic), 4);
magic = swap_endian(magic);
CAFFE_ENFORCE_EQ(magic, 2049, "Incorrect label file magic.");
image_file.read(reinterpret_cast<char*>(&num_items), 4);
num_items = swap_endian(num_items);
label_file.read(reinterpret_cast<char*>(&num_labels), 4);
num_labels = swap_endian(num_labels);
CAFFE_ENFORCE_EQ(num_items, num_labels);
image_file.read(reinterpret_cast<char*>(&rows), 4);
rows = swap_endian(rows);
image_file.read(reinterpret_cast<char*>(&cols), 4);
cols = swap_endian(cols);
// leveldb
std::unique_ptr<db::DB> mnist_db(db::CreateDB(FLAGS_db, db_path, db::NEW));
std::unique_ptr<db::Transaction> transaction(mnist_db->NewTransaction());
// Storing to db
char label_value;
std::vector<char> pixels(rows * cols);
int count = 0;
const int kMaxKeyLength = 11;
char key_cstr[kMaxKeyLength];
TensorProtos protos;
TensorProto* data = protos.add_protos();
TensorProto* label = protos.add_protos();
data->set_data_type(TensorProto::BYTE);
if (FLAGS_channel_first) {
data->add_dims(1);
data->add_dims(rows);
data->add_dims(cols);
} else {
data->add_dims(rows);
data->add_dims(cols);
data->add_dims(1);
}
label->set_data_type(TensorProto::INT32);
label->add_int32_data(0);
LOG(INFO) << "A total of " << num_items << " items.";
LOG(INFO) << "Rows: " << rows << " Cols: " << cols;
for (int item_id = 0; item_id < num_items; ++item_id) {
image_file.read(pixels.data(), rows * cols);
label_file.read(&label_value, 1);
for (int i = 0; i < rows * cols; ++i) {
data->set_byte_data(pixels.data(), rows * cols);
}
label->set_int32_data(0, static_cast<int>(label_value));
snprintf(key_cstr, kMaxKeyLength, "%08d", item_id);
string keystr(key_cstr);
// Put in db
transaction->Put(keystr, protos.SerializeAsString());
if (++count % 1000 == 0) {
transaction->Commit();
}
if (data_limit > 0 && count == data_limit) {
LOG(INFO) << "Reached data limit of " << data_limit << ", stop.";
break;
}
}
}
} // namespace caffe2
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::convert_dataset(
FLAGS_image_file.c_str(),
FLAGS_label_file.c_str(),
FLAGS_output_file.c_str(),
FLAGS_data_limit);
return 0;
}

57
binaries/predictor_verifier.cc
View File

@ -1,57 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "caffe2/core/flags.h"
#include "caffe2/core/init.h"
#include "caffe2/predictor/predictor.h"
#include "caffe2/utils/proto_utils.h"
C10_DEFINE_string(init_net, "", "The given path to the init protobuffer.");
C10_DEFINE_string(
predict_net,
"",
"The given path to the predict protobuffer.");
namespace caffe2 {
void run() {
if (FLAGS_init_net.empty()) {
LOG(FATAL) << "No init net specified. Use --init_net=/path/to/net.";
}
if (FLAGS_predict_net.empty()) {
LOG(FATAL) << "No predict net specified. Use --predict_net=/path/to/net.";
}
caffe2::NetDef init_net, predict_net;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net));
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predict_net));
// Can be large due to constant fills
VLOG(1) << "Init net: " << ProtoDebugString(init_net);
LOG(INFO) << "Predict net: " << ProtoDebugString(predict_net);
auto predictor = std::make_unique<Predictor>(init_net, predict_net);
LOG(INFO) << "Checking that a null forward-pass works";
Predictor::TensorList inputVec, outputVec;
(*predictor)(inputVec, &outputVec);
CAFFE_ENFORCE_GT(outputVec.size(), 0);
}
}
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::run();
// This is to allow us to use memory leak checks.
caffe2::ShutdownProtobufLibrary();
return 0;
}

70
binaries/print_registered_core_operators.cc
View File

@ -1,70 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <iostream>
#include <string>
#include "caffe2/core/init.h"
#include "caffe2/core/operator.h"
#include "caffe2/core/operator_schema.h"
C10_DEFINE_string(schema, "", "Print doc and schema of a particular operator");
static bool HasSchema(const std::string& str) {
return caffe2::OpSchemaRegistry::Schema(str);
}
static bool HasDoc(const std::string& str) {
const auto* schema = caffe2::OpSchemaRegistry::Schema(str);
return (schema != nullptr) && (schema->doc() != nullptr);
}
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
if (!FLAGS_schema.empty()) {
const auto* schema = caffe2::OpSchemaRegistry::Schema(FLAGS_schema);
if (!schema) {
std::cerr << "Operator " << FLAGS_schema << " doesn't have a schema"
<< std::endl;
return 1;
}
std::cout << "Operator " << FLAGS_schema << ": " << std::endl << *schema;
return 0;
}
for (const auto& pair : *caffe2::gDeviceTypeRegistry()) {
std::cout << "Device type " << pair.first
#ifndef CAFFE2_USE_LITE_PROTO
<< " ("
<< at::DeviceTypeName(static_cast<caffe2::DeviceType>(pair.first))
<< ")"
#endif
<< std::endl;
for (const auto& key : pair.second->Keys()) {
std::cout << "\t(schema: " << HasSchema(key) << ", doc: " << HasDoc(key)
<< ")\t" << key << std::endl;
}
}
std::cout << "Operators that have gradients registered:" << std::endl;
for (const auto& key : caffe2::GradientRegistry()->Keys()) {
std::cout << "\t(schema: " << HasSchema(key) << ", doc: "
<< HasDoc(key) << ")\t"
<< key << std::endl;
}
return 0;
}

40
binaries/run_plan.cc
View File

@ -1,40 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "caffe2/core/init.h"
#include "caffe2/core/operator.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/utils/proto_utils.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(plan, "", "The given path to the plan protobuffer.");
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
if (FLAGS_plan.size() == 0) {
LOG(ERROR) << "No plan specified. Use --plan=/path/to/plan.";
return 0;
}
LOG(INFO) << "Loading plan: " << FLAGS_plan;
caffe2::PlanDef plan_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_plan, &plan_def));
std::unique_ptr<caffe2::Workspace> workspace(new caffe2::Workspace());
workspace->RunPlan(plan_def);
// This is to allow us to use memory leak checks.
caffe2::ShutdownProtobufLibrary();
return 0;
}

49
binaries/run_plan_mpi.cc
View File

@ -1,49 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <mpi.h>
#include "c10/util/Flags.h"
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/operator.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/utils/proto_utils.h"
C10_DEFINE_string(plan, "", "The given path to the plan protobuffer.");
int main(int argc, char** argv) {
c10::SetUsageMessage("Runs a caffe2 plan that has MPI operators in it.");
int mpi_ret;
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_ret);
if (mpi_ret != MPI_THREAD_MULTIPLE &&
mpi_ret != MPI_THREAD_SERIALIZED) {
std::cerr << "Caffe2 MPI requires the underlying MPI to support the "
"MPI_THREAD_SERIALIZED or MPI_THREAD_MULTIPLE mode.\n";
return 1;
}
caffe2::GlobalInit(&argc, &argv);
LOG(INFO) << "Loading plan: " << FLAGS_plan;
caffe2::PlanDef plan_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_plan, &plan_def));
std::unique_ptr<caffe2::Workspace> workspace(new caffe2::Workspace());
workspace->RunPlan(plan_def);
// This is to allow us to use memory leak checks.
caffe2::ShutdownProtobufLibrary();
MPI_Finalize();
return 0;
}

210
binaries/speed_benchmark.cc
View File

@ -1,210 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <string>
#include "caffe2/core/blob_serialization.h"
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/operator.h"
#include "caffe2/core/tensor_int8.h"
#ifdef CAFFE2_OPTIMIZER
#include "caffe2/opt/optimizer.h"
#endif
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/utils/proto_utils.h"
#include "caffe2/utils/string_utils.h"
C10_DEFINE_string(net, "", "The given net to benchmark.");
C10_DEFINE_string(init_net, "", "The given net to initialize any parameters.");
C10_DEFINE_string(
input,
"",
"Input that is needed for running the network. If "
"multiple input needed, use comma separated string.");
C10_DEFINE_string(
input_file,
"",
"Input file that contain the serialized protobuf for "
"the input blobs. If multiple input needed, use comma "
"separated string. Must have the same number of items "
"as input does.");
C10_DEFINE_string(
input_dims,
"",
"Alternate to input_files, if all inputs are simple "
"float TensorCPUs, specify the dimension using comma "
"separated numbers. If multiple input needed, use "
"semicolon to separate the dimension of different "
"tensors.");
C10_DEFINE_string(input_type, "", "Input type (uint8_t/float)");
C10_DEFINE_string(
output,
"",
"Output that should be dumped after the execution "
"finishes. If multiple outputs are needed, use comma "
"separated string. If you want to dump everything, pass "
"'*' as the output value.");
C10_DEFINE_string(
output_folder,
"",
"The folder that the output should be written to. This "
"folder must already exist in the file system.");
C10_DEFINE_int(warmup, 0, "The number of iterations to warm up.");
C10_DEFINE_int(iter, 10, "The number of iterations to run.");
C10_DEFINE_int(opt, 0, "The level of optimization to run automatically.");
C10_DEFINE_bool(
run_individual,
false,
"Whether to benchmark individual operators.");
C10_DEFINE_bool(force_engine, false, "Force engine field for all operators");
C10_DEFINE_string(engine, "", "Forced engine field value");
C10_DEFINE_bool(force_algo, false, "Force algo arg for all operators");
C10_DEFINE_string(algo, "", "Forced algo arg value");
using std::string;
using std::unique_ptr;
using std::vector;
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
unique_ptr<caffe2::Workspace> workspace(new caffe2::Workspace());
// Run initialization network.
caffe2::NetDef net_def;
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &net_def));
CAFFE_ENFORCE(workspace->RunNetOnce(net_def));
// Load input.
if (FLAGS_input.size()) {
vector<string> input_names = caffe2::split(',', FLAGS_input);
if (FLAGS_input_file.size()) {
vector<string> input_files = caffe2::split(',', FLAGS_input_file);
CAFFE_ENFORCE_EQ(
input_names.size(),
input_files.size(),
"Input name and file should have the same number.");
for (int i = 0; i < input_names.size(); ++i) {
caffe2::BlobProto blob_proto;
CAFFE_ENFORCE(caffe2::ReadProtoFromFile(input_files[i], &blob_proto));
DeserializeBlob(blob_proto, workspace->CreateBlob(input_names[i]));
}
} else if (FLAGS_input_dims.size() || FLAGS_input_type.size()) {
CAFFE_ENFORCE_GE(
FLAGS_input_dims.size(),
0,
"Input dims must be specified when input tensors are used.");
CAFFE_ENFORCE_GE(
FLAGS_input_type.size(),
0,
"Input type must be specified when input tensors are used.");
vector<string> input_dims_list = caffe2::split(';', FLAGS_input_dims);
CAFFE_ENFORCE_EQ(
input_names.size(),
input_dims_list.size(),
"Input name and dims should have the same number of items.");
vector<string> input_type_list = caffe2::split(';', FLAGS_input_type);
CAFFE_ENFORCE_EQ(
input_names.size(),
input_type_list.size(),
"Input name and type should have the same number of items.");
for (size_t i = 0; i < input_names.size(); ++i) {
vector<string> input_dims_str = caffe2::split(',', input_dims_list[i]);
vector<int> input_dims;
for (const string& s : input_dims_str) {
input_dims.push_back(std::stoi(s));
}
caffe2::Blob* blob = workspace->GetBlob(input_names[i]);
if (blob == nullptr) {
blob = workspace->CreateBlob(input_names[i]);
}
if (input_type_list[i] == "uint8_t") {
caffe2::int8::Int8TensorCPU* tensor =
blob->GetMutable<caffe2::int8::Int8TensorCPU>();
TORCH_CHECK_NOTNULL(tensor);
tensor->t.Resize(input_dims);
tensor->t.mutable_data<uint8_t>();
} else if (input_type_list[i] == "float") {
caffe2::TensorCPU* tensor = BlobGetMutableTensor(blob, caffe2::CPU);
TORCH_CHECK_NOTNULL(tensor);
tensor->Resize(input_dims);
tensor->mutable_data<float>();
} else {
CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
}
}
} else {
CAFFE_THROW(
"You requested input tensors, but neither input_file nor "
"input_dims is set.");
}
}
// Run main network.
CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_net, &net_def));
if (!net_def.has_name()) {
net_def.set_name("benchmark");
}
// force changing engine and algo
if (FLAGS_force_engine) {
LOG(INFO) << "force engine be: " << FLAGS_engine;
for (const auto& op : net_def.op()) {
const_cast<caffe2::OperatorDef*>(&op)->set_engine(FLAGS_engine);
}
}
if (FLAGS_force_algo) {
LOG(INFO) << "force algo be: " << FLAGS_algo;
for (const auto& op : net_def.op()) {
caffe2::GetMutableArgument(
"algo", true, const_cast<caffe2::OperatorDef*>(&op))
->set_s(FLAGS_algo);
}
}
if (FLAGS_opt) {
#ifdef CAFFE2_OPTIMIZER
net_def = caffe2::opt::optimize(net_def, workspace.get(), FLAGS_opt);
#else
LOG(WARNING) << "Caffe2 not compiled with optimization passes.";
#endif
}
caffe2::NetBase* net = workspace->CreateNet(net_def);
TORCH_CHECK_NOTNULL(net);
CAFFE_ENFORCE(net->Run());
net->TEST_Benchmark(FLAGS_warmup, FLAGS_iter, FLAGS_run_individual);
string output_prefix =
FLAGS_output_folder.size() ? FLAGS_output_folder + "/" : "";
if (FLAGS_output.size()) {
vector<string> output_names = caffe2::split(',', FLAGS_output);
if (FLAGS_output == "*") {
output_names = workspace->Blobs();
}
for (const string& name : output_names) {
CAFFE_ENFORCE(
workspace->HasBlob(name),
"You requested a non-existing blob: ",
name);
string serialized = SerializeBlob(*workspace->GetBlob(name), name);
string output_filename = output_prefix + name;
caffe2::WriteStringToFile(serialized, output_filename.c_str());
}
}
return 0;
}

77
binaries/split_db.cc
View File

@ -1,77 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <string>
#include <sstream>
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/proto/caffe2_pb.h"
#include "caffe2/core/logging.h"
C10_DEFINE_string(input_db, "", "The input db.");
C10_DEFINE_int(splits, 0, "The number of splits.");
C10_DEFINE_string(db_type, "", "The db type.");
C10_DEFINE_int(batch_size, 1000, "The write batch size.");
namespace caffe2 {
static int Split(int argc, char** argv) {
GlobalInit(&argc, &argv);
CAFFE_ENFORCE(FLAGS_input_db.size(), "Must specify --input_db=/path/to/db.");
CAFFE_ENFORCE(FLAGS_splits > 0, "Must specify a nonnegative split number.");
CAFFE_ENFORCE(FLAGS_db_type.size(), "Must specify --db_type=[a db type].");
unique_ptr<db::DB> in_db(
db::CreateDB(FLAGS_db_type, FLAGS_input_db, db::READ));
CAFFE_ENFORCE(in_db != nullptr, "Cannot open input db: ", FLAGS_input_db);
unique_ptr<db::Cursor> cursor(in_db->NewCursor());
// This usually won't happen, but FWIW.
CAFFE_ENFORCE(
cursor != nullptr, "Cannot obtain cursor for input db: ", FLAGS_input_db);
vector<unique_ptr<db::DB>> out_dbs;
vector<unique_ptr<db::Transaction>> transactions;
for (int i = 0; i < FLAGS_splits; ++i) {
out_dbs.push_back(unique_ptr<db::DB>(db::CreateDB(
FLAGS_db_type, FLAGS_input_db + "_split_" + to_string(i), db::NEW)));
CAFFE_ENFORCE(out_dbs.back().get(), "Cannot create output db #", i);
transactions.push_back(
unique_ptr<db::Transaction>(out_dbs[i]->NewTransaction()));
CAFFE_ENFORCE(
transactions.back().get(), "Cannot get transaction for output db #", i);
}
int count = 0;
for (; cursor->Valid(); cursor->Next()) {
transactions[count % FLAGS_splits]->Put(cursor->key(), cursor->value());
if (++count % FLAGS_batch_size == 0) {
for (int i = 0; i < FLAGS_splits; ++i) {
transactions[i]->Commit();
}
LOG(INFO) << "Split " << count << " items so far.";
}
}
LOG(INFO) << "A total of " << count << " items processed.";
return 0;
}
} // namespace caffe2
int main(int argc, char** argv) {
return caffe2::Split(argc, argv);
}

89
binaries/tutorial_blob.cc
View File

@ -1,89 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "caffe2/core/blob.h"
#include "caffe2/core/init.h"
#include "caffe2/core/tensor.h"
#include "caffe2/core/logging.h"
// We will be lazy and just use the whole namespace.
using namespace caffe2;
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
caffe2::ShowLogInfoToStderr();
LOG(INFO) <<
"This script corresponds to the Blob part of the Caffe2 C++ "
"tutorial.";
LOG(INFO) << "Let's create a blob myblob.";
Blob myblob;
LOG(INFO) << "Let's set it to int and set the value to 10.";
int* myint = myblob.GetMutable<int>();
*myint = 10;
LOG(INFO)
<< "Is the blob type int? "
<< myblob.IsType<int>();
LOG(INFO)
<< "Is the blob type float? "
<< myblob.IsType<float>();
const int& myint_const = myblob.Get<int>();
LOG(INFO)
<< "The value of the int number stored in the blob is: "
<< myint_const;
LOG(INFO)
<< "Let's try to get a float pointer. This will trigger an exception.";
try {
const float& myfloat = myblob.Get<float>();
LOG(FATAL) << "This line should never happen.";
} catch (std::exception& e) {
LOG(INFO)
<< "As expected, we got an exception. Its content says: "
<< e.what();
}
LOG(INFO) <<
"However, we can change the content type (and destroy the old "
"content) by calling GetMutable. Let's change it to double.";
double* mydouble = myblob.GetMutable<double>();
*mydouble = 3.14;
LOG(INFO) << "The new content is: " << myblob.Get<double>();
LOG(INFO) <<
"If we have a pre-created object, we can use Reset() to transfer the "
"object to a blob.";
std::string* pvec = new std::string();
myblob.Reset(pvec); // no need to release pvec, myblob takes ownership.
LOG(INFO) << "Is the blob now of type string? "
<< myblob.IsType<std::string>();
LOG(INFO) << "This concludes the blob tutorial.";
return 0;
}

63
binaries/zmq_feeder.cc
View File

@ -1,63 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This binary provides an easy way to open a zeromq server and feeds data to
// clients connect to it. It uses the Caffe2 db as the backend, thus allowing
// one to convert any db-compliant storage to a zeromq service.
#include "caffe2/core/db.h"
#include "caffe2/core/init.h"
#include "caffe2/core/logging.h"
#include "caffe2/utils/zmq_helper.h"
C10_DEFINE_string(server, "tcp://*:5555", "The server address.");
C10_DEFINE_string(input_db, "", "The input db.");
C10_DEFINE_string(input_db_type, "", "The input db type.");
int main(int argc, char** argv) {
caffe2::GlobalInit(&argc, &argv);
LOG(INFO) << "Opening DB...";
auto in_db = caffe2::db::CreateDB(
FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ);
CAFFE_ENFORCE(
in_db,
"Cannot load input db " + FLAGS_input_db + " of expected type " +
FLAGS_input_db_type);
auto cursor = in_db->NewCursor();
LOG(INFO) << "DB opened.";
LOG(INFO) << "Starting ZeroMQ server...";
// Socket to talk to clients
caffe2::ZmqSocket sender(ZMQ_PUSH);
sender.Bind(FLAGS_server);
LOG(INFO) << "Server created at " << FLAGS_server;
while (1) {
VLOG(1) << "Sending " << cursor->key();
sender.SendTillSuccess(cursor->key(), ZMQ_SNDMORE);
sender.SendTillSuccess(cursor->value(), 0);
cursor->Next();
if (!cursor->Valid()) {
cursor->SeekToFirst();
}
}
// We do not do an elegant quit since this binary is going to be terminated by
// control+C.
return 0;
}

2
caffe2/CMakeLists.txt
View File

@ -122,8 +122,6 @@ if(BUILD_CAFFE2 AND NOT INTERN_BUILD_MOBILE)
if(USE_NVRTC)
add_subdirectory(cuda_rtc)
endif()
add_subdirectory(db)
add_subdirectory(distributed)
add_subdirectory(ideep)
add_subdirectory(mobile)
add_subdirectory(mpi)

52
cmake/Dependencies.cmake
View File

@ -865,18 +865,6 @@ else()
caffe2_update_option(USE_FAKELOWP OFF)
endif()
# ---[ LMDB
if(USE_LMDB)
find_package(LMDB)
if(LMDB_FOUND)
include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
list(APPEND Caffe2_DEPENDENCY_LIBS ${LMDB_LIBRARIES})
else()
message(WARNING "Not compiling with LMDB. Suppress this warning with -DUSE_LMDB=OFF")
caffe2_update_option(USE_LMDB OFF)
endif()
endif()
if(USE_OPENCL)
message(INFO "USING OPENCL")
find_package(OpenCL REQUIRED)
@ -884,22 +872,6 @@ if(USE_OPENCL)
list(APPEND Caffe2_DEPENDENCY_LIBS ${OpenCL_LIBRARIES})
endif()
# ---[ LevelDB
# ---[ Snappy
if(USE_LEVELDB)
find_package(LevelDB)
find_package(Snappy)
if(LEVELDB_FOUND AND SNAPPY_FOUND)
include_directories(SYSTEM ${LevelDB_INCLUDE})
list(APPEND Caffe2_DEPENDENCY_LIBS ${LevelDB_LIBRARIES})
include_directories(SYSTEM ${Snappy_INCLUDE_DIR})
list(APPEND Caffe2_DEPENDENCY_LIBS ${Snappy_LIBRARIES})
else()
message(WARNING "Not compiling with LevelDB. Suppress this warning with -DUSE_LEVELDB=OFF")
caffe2_update_option(USE_LEVELDB OFF)
endif()
endif()
# ---[ NUMA
if(USE_NUMA)
if(LINUX)
@ -914,30 +886,6 @@ if(USE_NUMA)
endif()
endif()
# ---[ ZMQ
if(USE_ZMQ)
find_package(ZMQ)
if(ZMQ_FOUND)
include_directories(SYSTEM ${ZMQ_INCLUDE_DIR})
list(APPEND Caffe2_DEPENDENCY_LIBS ${ZMQ_LIBRARIES})
else()
message(WARNING "Not compiling with ZMQ. Suppress this warning with -DUSE_ZMQ=OFF")
caffe2_update_option(USE_ZMQ OFF)
endif()
endif()
# ---[ Redis
if(USE_REDIS)
find_package(Hiredis)
if(HIREDIS_FOUND)
include_directories(SYSTEM ${Hiredis_INCLUDE})
list(APPEND Caffe2_DEPENDENCY_LIBS ${Hiredis_LIBRARIES})
else()
message(WARNING "Not compiling with Redis. Suppress this warning with -DUSE_REDIS=OFF")
caffe2_update_option(USE_REDIS OFF)
endif()
endif()
if(USE_ITT)
find_package(ITT)
if(ITT_FOUND)

23
cmake/Modules/FindHiredis.cmake
View File

@ -1,23 +0,0 @@
# Find the Hiredis libraries
#
# The following variables are optionally searched for defaults
# HIREDIS_ROOT_DIR: Base directory where all Hiredis components are found
#
# The following are set after configuration is done:
# HIREDIS_FOUND
# Hiredis_INCLUDE_DIR
# Hiredis_LIBRARIES
find_path(Hiredis_INCLUDE_DIR NAMES hiredis/hiredis.h
PATHS ${HIREDIS_ROOT_DIR} ${HIREDIS_ROOT_DIR}/include)
find_library(Hiredis_LIBRARIES NAMES hiredis
PATHS ${HIREDIS_ROOT_DIR} ${HIREDIS_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Hiredis DEFAULT_MSG Hiredis_INCLUDE_DIR Hiredis_LIBRARIES)
if(HIREDIS_FOUND)
message(STATUS "Found Hiredis (include: ${Hiredis_INCLUDE_DIR}, library: ${Hiredis_LIBRARIES})")
mark_as_advanced(Hiredis_INCLUDE_DIR Hiredis_LIBRARIES)
endif()

32
cmake/Modules/FindLMDB.cmake
View File

@ -1,32 +0,0 @@
# Try to find the LMBD libraries and headers
# LMDB_FOUND - system has LMDB lib
# LMDB_INCLUDE_DIR - the LMDB include directory
# LMDB_LIBRARIES - Libraries needed to use LMDB
# FindCWD based on FindGMP by:
# Copyright (c) 2006, Laurent Montel, <montel@kde.org>
#
# Redistribution and use is allowed according to the terms of the BSD license.
# Adapted from FindCWD by:
# Copyright 2013 Conrad Steenberg <conrad.steenberg@gmail.com>
# Aug 31, 2013
if(MSVC)
find_package(LMDB NO_MODULE)
else()
find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS "$ENV{LMDB_DIR}/include")
find_library(LMDB_LIBRARIES NAMES lmdb PATHS "$ENV{LMDB_DIR}/lib" )
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LMDB DEFAULT_MSG LMDB_INCLUDE_DIR LMDB_LIBRARIES)
if(LMDB_FOUND)
message(STATUS "Found lmdb (include: ${LMDB_INCLUDE_DIR}, library: ${LMDB_LIBRARIES})")
mark_as_advanced(LMDB_INCLUDE_DIR LMDB_LIBRARIES)
caffe_parse_header(${LMDB_INCLUDE_DIR}/lmdb.h
LMDB_VERSION_LINES MDB_VERSION_MAJOR MDB_VERSION_MINOR MDB_VERSION_PATCH)
set(LMDB_VERSION "${MDB_VERSION_MAJOR}.${MDB_VERSION_MINOR}.${MDB_VERSION_PATCH}")
endif()

44
cmake/Modules/FindLevelDB.cmake
View File

@ -1,44 +0,0 @@
# - Find LevelDB
#
# LevelDB_INCLUDES - List of LevelDB includes
# LevelDB_LIBRARIES - List of libraries when using LevelDB.
# LevelDB_FOUND - True if LevelDB found.
# Look for the header file.
find_path(LevelDB_INCLUDE NAMES leveldb/db.h
PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include
DOC "Path in which the file leveldb/db.h is located." )
# Look for the library.
find_library(LevelDB_LIBRARY NAMES leveldb
PATHS /usr/lib $ENV{LEVELDB_ROOT}/lib
DOC "Path to leveldb library." )
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LevelDB DEFAULT_MSG LevelDB_INCLUDE LevelDB_LIBRARY)
if(LEVELDB_FOUND)
message(STATUS "Found LevelDB (include: ${LevelDB_INCLUDE}, library: ${LevelDB_LIBRARY})")
set(LevelDB_INCLUDES ${LevelDB_INCLUDE})
set(LevelDB_LIBRARIES ${LevelDB_LIBRARY})
mark_as_advanced(LevelDB_INCLUDE LevelDB_LIBRARY)
if(EXISTS "${LevelDB_INCLUDE}/leveldb/db.h")
file(STRINGS "${LevelDB_INCLUDE}/leveldb/db.h" __version_lines
REGEX "static const int k[^V]+Version[ \t]+=[ \t]+[0-9]+;")
foreach(__line ${__version_lines})
if(__line MATCHES "[^k]+kMajorVersion[ \t]+=[ \t]+([0-9]+);")
set(LEVELDB_VERSION_MAJOR ${CMAKE_MATCH_1})
elseif(__line MATCHES "[^k]+kMinorVersion[ \t]+=[ \t]+([0-9]+);")
set(LEVELDB_VERSION_MINOR ${CMAKE_MATCH_1})
endif()
endforeach()
if(LEVELDB_VERSION_MAJOR AND LEVELDB_VERSION_MINOR)
set(LEVELDB_VERSION "${LEVELDB_VERSION_MAJOR}.${LEVELDB_VERSION_MINOR}")
endif()
# caffe_clear_vars(__line __version_lines)
endif()
endif()

23
cmake/Modules/FindRocksDB.cmake
View File

@ -1,23 +0,0 @@
# Find the RocksDB libraries
#
# The following variables are optionally searched for defaults
# ROCKSDB_ROOT_DIR: Base directory where all RocksDB components are found
#
# The following are set after configuration is done:
# ROCKSDB_FOUND
# RocksDB_INCLUDE_DIR
# RocksDB_LIBRARIES
find_path(RocksDB_INCLUDE_DIR NAMES rocksdb/db.h
PATHS ${ROCKSDB_ROOT_DIR} ${ROCKSDB_ROOT_DIR}/include)
find_library(RocksDB_LIBRARIES NAMES rocksdb
PATHS ${ROCKSDB_ROOT_DIR} ${ROCKSDB_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(RocksDB DEFAULT_MSG RocksDB_INCLUDE_DIR RocksDB_LIBRARIES)
if(ROCKSDB_FOUND)
message(STATUS "Found RocksDB (include: ${RocksDB_INCLUDE_DIR}, library: ${RocksDB_LIBRARIES})")
mark_as_advanced(RocksDB_INCLUDE_DIR RocksDB_LIBRARIES)
endif()

26
cmake/Modules/FindSnappy.cmake
View File

@ -1,26 +0,0 @@
# Find the Snappy libraries
#
# The following variables are optionally searched for defaults
# SNAPPY_ROOT_DIR: Base directory where all Snappy components are found
#
# The following are set after configuration is done:
# SNAPPY_FOUND
# Snappy_INCLUDE_DIR
# Snappy_LIBRARIES
find_path(Snappy_INCLUDE_DIR NAMES snappy.h
PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/include)
find_library(Snappy_LIBRARIES NAMES snappy
PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_INCLUDE_DIR Snappy_LIBRARIES)
if(SNAPPY_FOUND)
message(STATUS "Found Snappy (include: ${Snappy_INCLUDE_DIR}, library: ${Snappy_LIBRARIES})")
mark_as_advanced(Snappy_INCLUDE_DIR Snappy_LIBRARIES)
caffe_parse_header(${Snappy_INCLUDE_DIR}/snappy-stubs-public.h
SNAPPY_VERION_LINES SNAPPY_MAJOR SNAPPY_MINOR SNAPPY_PATCHLEVEL)
set(Snappy_VERSION "${SNAPPY_MAJOR}.${SNAPPY_MINOR}.${SNAPPY_PATCHLEVEL}")
endif()

32
cmake/Modules/FindZMQ.cmake
View File

@ -1,32 +0,0 @@
# Find the ZMQ libraries
#
# The following variables are optionally searched for defaults
# ZMQ_ROOT_DIR: Base directory where all ZMQ components are found
#
# The following are set after configuration is done:
# ZMQ_FOUND
# ZMQ_INCLUDE_DIR
# ZMQ_LIBRARIES
# ZMQ_VERSION_MAJOR
find_path(ZMQ_INCLUDE_DIR NAMES zmq.h
PATHS ${ZMQ_ROOT_DIR} ${ZMQ_ROOT_DIR}/include)
find_library(ZMQ_LIBRARIES NAMES zmq
PATHS ${ZMQ_ROOT_DIR} ${ZMQ_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ZMQ DEFAULT_MSG ZMQ_INCLUDE_DIR ZMQ_LIBRARIES)
if(ZMQ_FOUND)
message(STATUS "Found ZMQ (include: ${ZMQ_INCLUDE_DIR}, library: ${ZMQ_LIBRARIES})")
mark_as_advanced(ZMQ_INCLUDE_DIR ZMQ_LIBRARIES)
caffe_parse_header(${ZMQ_INCLUDE_DIR}/zmq.h ZMQ_VERSION_LINES ZMQ_VERSION_MAJOR)
if(${ZMQ_VERSION_MAJOR} VERSION_LESS "3")
message(WARNING "Caffe2 requires zmq version 3 or above, but found " ${ZMQ_VERSION_MAJOR} ". Disabling zmq for now.")
set(ZMQ_FOUND)
else()
endif()
endif()

12
cmake/Summary.cmake
View File

@ -130,16 +130,7 @@ function(caffe2_print_configuration_summary)
message(STATUS " USE_KINETO : ${USE_KINETO}")
message(STATUS " USE_GFLAGS : ${USE_GFLAGS}")
message(STATUS " USE_GLOG : ${USE_GLOG}")
message(STATUS " USE_LEVELDB : ${USE_LEVELDB}")
if(${USE_LEVELDB})
message(STATUS " LevelDB version : ${LEVELDB_VERSION}")
message(STATUS " Snappy version : ${Snappy_VERSION}")
endif()
message(STATUS " USE_LITE_PROTO : ${USE_LITE_PROTO}")
message(STATUS " USE_LMDB : ${USE_LMDB}")
if(${USE_LMDB})
message(STATUS " LMDB version : ${LMDB_VERSION}")
endif()
message(STATUS " USE_METAL : ${USE_METAL}")
message(STATUS " USE_PYTORCH_METAL : ${USE_PYTORCH_METAL}")
message(STATUS " USE_PYTORCH_METAL_EXPORT : ${USE_PYTORCH_METAL_EXPORT}")
@ -178,9 +169,6 @@ function(caffe2_print_configuration_summary)
message(STATUS " USE_QNNPACK : ${USE_QNNPACK}")
message(STATUS " USE_PYTORCH_QNNPACK : ${USE_PYTORCH_QNNPACK}")
message(STATUS " USE_XNNPACK : ${USE_XNNPACK}")
message(STATUS " USE_REDIS : ${USE_REDIS}")
message(STATUS " USE_ROCKSDB : ${USE_ROCKSDB}")
message(STATUS " USE_ZMQ : ${USE_ZMQ}")
message(STATUS " USE_DISTRIBUTED : ${USE_DISTRIBUTED}")
if(${USE_DISTRIBUTED})
message(STATUS " USE_MPI : ${USE_MPI}")

1
modules/CMakeLists.txt
View File

@ -2,7 +2,6 @@ project(modules CXX C)
add_subdirectory(detectron)
add_subdirectory(module_test)
add_subdirectory(observers)
add_subdirectory(rocksdb)
# Finally, set Caffe2_MODULES to parent scope.
set(Caffe2_MODULES ${Caffe2_MODULES} PARENT_SCOPE)

78
modules/rocksdb/CMakeLists.txt
View File

@ -1,78 +0,0 @@
# ---[ RocksDB module
# In addition to being a useful module itself, RocksDB is also an exemplar
# case where show how one should built a Caffe2 module inside the Caffe2
# repository.
#
# This cmake file achieves two build modes:
# (1) If one is invoking the main Caffe2 build, we will check a USE_* option,
# in this case USE_ROCKSDB, to test if we want to build this module.
# (2) if we are building it in a standalone way, we will find the preinstalled
# Caffe2 library, and then build the library and install it.
# ---[ First, determine if we are building with the main repo or not.
# This is guarded by the CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO variable. It then
# routes build to two paths:
# (1) When we are building with the main repo, the caffe2_library is going to
# be already defined, and all related paths will be defined too. So we will
# simply test if the main repo build wants to build this module, in our
# case by the variable "USE_ROCKSDB".
# (2) When we are not building with the main repo, we will need to do the usual
# cmake setup: version checks, project options, find dependent packages,
# etc.
if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
if(NOT USE_ROCKSDB)
return()
endif()
else()
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(caffe2_rocksdb CXX)
find_package(Caffe2 REQUIRED)
option(BUILD_SHARED_LIBS "Build shared libs." ON)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../cmake/Modules)
endif()
# ---[ Second, find dependencies.
# This one should be similar to the standard dependency discovery in normal
# cmake. Note that for modules that are located in the Caffe2 repository,
# cmake related files, such as FindRocksDB in this case, should live in the
# cmake/ folder under root.
find_package(RocksDB CONFIG)
if((DEFINED RocksDB_DIR) AND RocksDB_DIR)
list(APPEND RocksDB_LIBRARIES RocksDB::rocksdb)
else()
message("RocksDB config not found. Fallback to legacy find.")
find_package(RocksDB)
if(NOT ROCKSDB_FOUND)
message(
FATAL_ERROR
"RocksDB not found. If you do not need caffe2_rocksdb, set "
"-DUSE_ROCKSDB=OFF to solve this error.")
endif()
endif()
# ---[ Third, create the CMake target.
# The key to note is that this library will need to depend on caffe2_library,
# which is the main lib of Caffe2. If your library explicitly depends on cuda,
# then you will need to depend on the caffe2_gpu_library as well.
add_library(caffe2_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb.cc)
# RocksDB 7 uses C++17 STL in header.
if(RocksDB_VERSION_MAJOR VERSION_GREATER_EQUAL 7)
set_target_properties(caffe2_rocksdb PROPERTIES CXX_STANDARD 17)
endif()
target_link_libraries(caffe2_rocksdb PUBLIC torch_library)
target_link_libraries(caffe2_rocksdb PRIVATE ${RocksDB_LIBRARIES})
target_include_directories(caffe2_rocksdb PRIVATE ${RocksDB_INCLUDE_DIR})
install(TARGETS caffe2_rocksdb DESTINATION lib)
# ---[ Last, Append the library to Caffe2_MODULES, if we are building with
# the main repo.
# The purpose of this is that, for all binaries built in the Caffe2 main repo,
# they will be built with the first class modules that are built. As a result,
# these binaries will not need to explicitly load these modules before using
# them.
# Note(jiayq): this also depends on a separate cmake move to reorg test builds
# and binary builds after modules. When it is done, this note should be removed.
if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
set(Caffe2_MODULES ${Caffe2_MODULES} caffe2_rocksdb PARENT_SCOPE)
endif()

118
modules/rocksdb/rocksdb.cc
View File

@ -1,118 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "caffe2/core/db.h"
#include "caffe2/core/logging.h"
#include "caffe2/core/module.h"
#include "caffe2/core/flags.h"
#include "rocksdb/db.h"
#include "rocksdb/utilities/leveldb_options.h"
C10_DEFINE_int(
caffe2_rocksdb_block_size,
65536,
"The caffe2 rocksdb block size when writing a rocksdb.");
namespace caffe2 {
namespace db {
class RocksDBCursor : public Cursor {
public:
explicit RocksDBCursor(rocksdb::DB* db)
: iter_(db->NewIterator(rocksdb::ReadOptions())) {
SeekToFirst();
}
~RocksDBCursor() {}
void Seek(const string& key) override { iter_->Seek(key); }
bool SupportsSeek() override { return true; }
void SeekToFirst() override { iter_->SeekToFirst(); }
void Next() override { iter_->Next(); }
string key() override { return iter_->key().ToString(); }
string value() override { return iter_->value().ToString(); }
bool Valid() override { return iter_->Valid(); }
private:
std::unique_ptr<rocksdb::Iterator> iter_;
};
class RocksDBTransaction : public Transaction {
public:
explicit RocksDBTransaction(rocksdb::DB* db) : db_(db) {
CAFFE_ENFORCE(db_);
batch_.reset(new rocksdb::WriteBatch());
}
~RocksDBTransaction() { Commit(); }
void Put(const string& key, string&& value) override {
batch_->Put(key, value);
}
void Commit() override {
rocksdb::Status status = db_->Write(rocksdb::WriteOptions(), batch_.get());
batch_.reset(new rocksdb::WriteBatch());
CAFFE_ENFORCE(
status.ok(), "Failed to write batch to rocksdb: " + status.ToString());
}
private:
rocksdb::DB* db_;
std::unique_ptr<rocksdb::WriteBatch> batch_;
C10_DISABLE_COPY_AND_ASSIGN(RocksDBTransaction);
};
class RocksDB : public DB {
public:
RocksDB(const string& source, Mode mode) : DB(source, mode) {
rocksdb::LevelDBOptions options;
options.block_size = FLAGS_caffe2_rocksdb_block_size;
options.write_buffer_size = 268435456;
options.max_open_files = 100;
options.error_if_exists = mode == NEW;
options.create_if_missing = mode != READ;
rocksdb::Options rocksdb_options = rocksdb::ConvertOptions(options);
rocksdb::DB* db_temp;
rocksdb::Status status = rocksdb::DB::Open(
rocksdb_options, source, &db_temp);
CAFFE_ENFORCE(
status.ok(),
"Failed to open rocksdb ",
source,
"\n",
status.ToString());
db_.reset(db_temp);
VLOG(1) << "Opened rocksdb " << source;
}
void Close() override { db_.reset(); }
unique_ptr<Cursor> NewCursor() override {
return make_unique<RocksDBCursor>(db_.get());
}
unique_ptr<Transaction> NewTransaction() override {
return make_unique<RocksDBTransaction>(db_.get());
}
private:
std::unique_ptr<rocksdb::DB> db_;
};
REGISTER_CAFFE2_DB(RocksDB, RocksDB);
// For lazy-minded, one can also call with lower-case name.
REGISTER_CAFFE2_DB(rocksdb, RocksDB);
} // namespace db
CAFFE2_MODULE(caffe2_rocksdb, "RocksDB implementation for caffe2::DB.");
} // namespace caffe2

2
scripts/build_android.sh
View File

@ -128,8 +128,6 @@ CMAKE_ARGS+=("-DUSE_CUDA=OFF")
CMAKE_ARGS+=("-DUSE_ITT=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
CMAKE_ARGS+=("-DUSE_LMDB=OFF")
CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
# Only toggle if VERBOSE=1

2
scripts/build_ios.sh
View File

@ -107,8 +107,6 @@ CMAKE_ARGS+=("-DUSE_CUDA=OFF")
CMAKE_ARGS+=("-DUSE_ITT=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
CMAKE_ARGS+=("-DUSE_LMDB=OFF")
CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_NUMPY=OFF")
CMAKE_ARGS+=("-DUSE_NNPACK=OFF")

2
scripts/build_mobile.sh
View File

@ -68,8 +68,6 @@ CMAKE_ARGS+=("-DUSE_CUDA=OFF")
CMAKE_ARGS+=("-DUSE_ITT=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
CMAKE_ARGS+=("-DUSE_LMDB=OFF")
CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
CMAKE_ARGS+=("-DUSE_MKLDNN=OFF")

3
scripts/build_tegra_x1.sh
View File

@ -29,10 +29,7 @@ sudo apt-get install \
# obtain optional dependencies that are usually useful to have.
echo "Installing optional dependencies."
sudo apt-get install \
libleveldb-dev \
liblmdb-dev \
libpython-dev \
libsnappy-dev \
python-numpy \
python-pip \
python-protobuf

7
scripts/build_tizen.sh
View File

@ -55,9 +55,6 @@ cmake .. \
-DRUN_HAVE_POSIX_REGEX=0 \
-DHAVE_GNU_POSIX_REGEX=0 \
-DUSE_MPI=OFF -DUSE_OPENMP=OFF \
-DUSE_ROCKSDB=OFF \
-DUSE_LEVELDB=OFF \
-DUSE_LMDB=OFF \
-DBUILD_PYTHON=OFF \
-DUSE_GLOO=OFF \
-DUSE_OPENCV=OFF \
@ -84,10 +81,7 @@ sudo zypper install \
# Obtain optional dependencies that are usually useful to have.
echo "Installing optional dependencies."
sudo zypper install \
libleveldb-dev \
liblmdb-dev \
libpython-dev \
libsnappy-dev \
python-numpy \
python-pip \
python-protobuf
@ -110,7 +104,6 @@ cmake "$CAFFE2_ROOT" \
-DUSE_CUDA=OFF \
-DUSE_ITT=OFF \
-DUSE_OPENCV=OFF \
-DUSE_LMDB=OFF \
-DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \
|| exit 1