mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 12:54:11 +08:00
7d5f7ed270
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/12714 This is a short change to enable c10 namespace in caffe2. We did not enable it before due to gflags global variable confusion, but it should have been mostly cleaned now. Right now, the plan on record is that namespace caffe2 and namespace aten will fully be supersets of namespace c10. Most of the diff is codemod, and only two places of non-codemod is in caffe2/core/common.h, where ``` using namespace c10; ``` is added, and in Flags.h, where instead of creating aliasing variables in c10 namespace, we directly put it in the global namespace to match gflags (and same behavior if gflags is not being built with). Reviewed By: dzhulgakov Differential Revision: D10390486 fbshipit-source-id: 5e2df730e28e29a052f513bddc558d9f78a23b9b
78 lines
2.7 KiB
C++
78 lines
2.7 KiB
C++
/**
|
|
* Copyright (c) 2016-present, Facebook, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
#include <string>
|
|
#include <sstream>
|
|
|
|
#include "caffe2/core/db.h"
|
|
#include "caffe2/core/init.h"
|
|
#include "caffe2/proto/caffe2_pb.h"
|
|
#include "caffe2/core/logging.h"
|
|
|
|
C10_DEFINE_string(input_db, "", "The input db.");
|
|
C10_DEFINE_int(splits, 0, "The number of splits.");
|
|
C10_DEFINE_string(db_type, "", "The db type.");
|
|
C10_DEFINE_int(batch_size, 1000, "The write batch size.");
|
|
|
|
namespace caffe2 {
|
|
|
|
static int Split(int argc, char** argv) {
|
|
GlobalInit(&argc, &argv);
|
|
|
|
CAFFE_ENFORCE(FLAGS_input_db.size(), "Must specify --input_db=/path/to/db.");
|
|
CAFFE_ENFORCE(FLAGS_splits > 0, "Must specify a nonnegative split number.");
|
|
CAFFE_ENFORCE(FLAGS_db_type.size(), "Must specify --db_type=[a db type].");
|
|
|
|
unique_ptr<db::DB> in_db(
|
|
db::CreateDB(FLAGS_db_type, FLAGS_input_db, db::READ));
|
|
CAFFE_ENFORCE(in_db != nullptr, "Cannot open input db: ", FLAGS_input_db);
|
|
unique_ptr<db::Cursor> cursor(in_db->NewCursor());
|
|
// This usually won't happen, but FWIW.
|
|
CAFFE_ENFORCE(
|
|
cursor != nullptr, "Cannot obtain cursor for input db: ", FLAGS_input_db);
|
|
|
|
vector<unique_ptr<db::DB>> out_dbs;
|
|
vector<unique_ptr<db::Transaction>> transactions;
|
|
for (int i = 0; i < FLAGS_splits; ++i) {
|
|
out_dbs.push_back(unique_ptr<db::DB>(db::CreateDB(
|
|
FLAGS_db_type, FLAGS_input_db + "_split_" + to_string(i), db::NEW)));
|
|
CAFFE_ENFORCE(out_dbs.back().get(), "Cannot create output db #", i);
|
|
transactions.push_back(
|
|
unique_ptr<db::Transaction>(out_dbs[i]->NewTransaction()));
|
|
CAFFE_ENFORCE(
|
|
transactions.back().get(), "Cannot get transaction for output db #", i);
|
|
}
|
|
|
|
int count = 0;
|
|
for (; cursor->Valid(); cursor->Next()) {
|
|
transactions[count % FLAGS_splits]->Put(cursor->key(), cursor->value());
|
|
if (++count % FLAGS_batch_size == 0) {
|
|
for (int i = 0; i < FLAGS_splits; ++i) {
|
|
transactions[i]->Commit();
|
|
}
|
|
LOG(INFO) << "Split " << count << " items so far.";
|
|
}
|
|
}
|
|
LOG(INFO) << "A total of " << count << " items processed.";
|
|
return 0;
|
|
}
|
|
|
|
} // namespace caffe2
|
|
|
|
int main(int argc, char** argv) {
|
|
return caffe2::Split(argc, argv);
|
|
}
|