experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add a mechanism for hiding, skipping, and modifying the generated op

Browse Source 
functions for C++.  A souped-up version of the hidden_ops mechanism in
Python, the intent is to use this for most or all of the client
languages, with a common list of changes to make in a common file and
per-language overrides.

Also:
* include the documentation for outputs in the generated comments
* several updates to C++ API to match Python
* fix C++ shape function for ConcatV2 now that we use it by default
* split op_gen_lib out of core:framework, since it is only used by
  the op generators, and I don't want to add another proto to
  mobile builds
Change: 146267344
This commit is contained in:
A. Unique TensorFlower 2017-02-01 11:23:07 -08:00 committed by TensorFlower Gardener
parent 287e845c52
commit 8fe32029f7
21 changed files with 557 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tensorflow/cc/BUILD
View File

@ -228,6 +228,7 @@ cc_library(
srcs = ["gradients/array_grad.cc"],
deps = [
":cc_ops",
":cc_ops_internal",
":grad_op_registry",
":gradients",
],
@ -239,6 +240,7 @@ tf_cc_test(
deps = [
":array_grad",
":cc_ops",
":cc_ops_internal",
":grad_op_registry",
":grad_testutil",
":gradient_checker",
@ -334,6 +336,7 @@ tf_gen_op_wrappers_cc(
"ops/const_op.h",
"ops/standard_ops.h",
],
override_file = "ops/op_gen_overrides.pbtxt",
pkg = "//tensorflow/core",
)
@ -387,6 +390,7 @@ cc_library_with_android_deps(
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:op_gen_lib",
"//tensorflow/core:proto_text",
"//tensorflow/core:protos_all_cc",
],

224
tensorflow/cc/framework/cc_op_gen.cc
View File

@ -15,6 +15,7 @@ limitations under the License.
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "tensorflow/cc/framework/cc_op_gen.h"
#include "tensorflow/core/framework/attr_value_util.h"
@ -38,7 +39,7 @@ const int kRightMargin = 79;
// Converts:
// bazel-out/.../genfiles/(external/YYY/)?XX
// to: XX.
string GetPath(const std::string& dot_h_fname) {
string GetPath(const string& dot_h_fname) {
auto pos = dot_h_fname.find("/genfiles/");
string result = dot_h_fname;
if (pos != string::npos) {
@ -60,7 +61,7 @@ string GetPath(const std::string& dot_h_fname) {
// cc/ops/gen_foo_ops.h
// to:
// CC_OPS_GEN_FOO_OPS_H_
string ToGuard(const std::string& path) {
string ToGuard(const string& path) {
string guard;
guard.reserve(path.size() + 1); // + 1 -> trailing _
for (const char c : path) {
@ -360,7 +361,12 @@ bool HasOptionalAttrs(
}
struct OpInfo {
explicit OpInfo(const OpDef& op_def);
// graph_op_def: The OpDef used by the runtime, has the names that
// must be used when calling NodeBuilder.
// interface_op_def: The OpDef used in the interface in the generated
// code, with possibly overridden names and defaults.
explicit OpInfo(const OpDef& graph_op_def, const OpDef& inteface_op_def,
const std::vector<string>& aliases);
string GetOpAttrStruct() const;
string GetConstructorDecl(StringPiece op_name_prefix,
bool include_attr) const;
@ -378,11 +384,15 @@ struct OpInfo {
bool has_optional_attrs;
string comment;
const OpDef& graph_op_def;
const OpDef& op_def;
const std::vector<string>& aliases;
std::unordered_map<string, string> inferred_input_attrs;
};
OpInfo::OpInfo(const OpDef& op_def) : op_def(op_def) {
OpInfo::OpInfo(const OpDef& g_op_def, const OpDef& i_op_def,
const std::vector<string>& a)
: graph_op_def(g_op_def), op_def(i_op_def), aliases(a) {
op_name = op_def.name();
InferOpAttributes(op_def, &inferred_input_attrs);
has_optional_attrs = HasOptionalAttrs(op_def, inferred_input_attrs);
@ -443,7 +453,6 @@ OpInfo::OpInfo(const OpDef& op_def) : op_def(op_def) {
strings::StrAppend(&comment, " ", attr.description(), "\n");
}
}
comment = MakeComment(comment, "");
for (int i = 0; i < op_def.output_arg_size(); ++i) {
const auto& arg = op_def.output_arg(i);
@ -453,13 +462,55 @@ OpInfo::OpInfo(const OpDef& op_def) : op_def(op_def) {
output_names.push_back(AvoidCPPKeywords(arg.name()));
is_list_output.push_back(is_list);
}
strings::StrAppend(&comment, "\nReturns:\n");
if (op_def.output_arg_size() == 0) { // No outputs.
strings::StrAppend(&comment, "* the created `Operation`\n");
} else if (op_def.output_arg_size() == 1) { // One output
if (is_list_output[0]) {
strings::StrAppend(&comment, "* `OutputList`: ");
} else {
strings::StrAppend(&comment, "* `Output`: ");
}
if (op_def.output_arg(0).description().empty()) {
strings::StrAppend(&comment, "The ", op_def.output_arg(0).name(),
" tensor.\n");
} else {
// TODO(josh11b): Word wrap this.
strings::StrAppend(&comment, op_def.output_arg(0).description(), "\n");
}
} else { // Multiple outputs.
for (int i = 0; i < op_def.output_arg_size(); ++i) {
if (is_list_output[i]) {
strings::StrAppend(&comment, "* `OutputList`");
} else {
strings::StrAppend(&comment, "* `Output`");
}
strings::StrAppend(&comment, " ", output_names[i]);
if (op_def.output_arg(i).description().empty()) {
strings::StrAppend(&comment, "\n");
} else {
// TODO(josh11b): Word wrap this.
strings::StrAppend(&comment, ": ", op_def.output_arg(i).description(),
"\n");
}
}
}
if (!aliases.empty()) {
strings::StrAppend(&comment, "\nAliases:\n");
for (const auto& alias : aliases) {
strings::StrAppend(&comment, "* ", alias, "\n");
}
}
comment = MakeComment(comment, "");
}
string OpInfo::GetOpAttrStruct() const {
string struct_fields;
string setters;
string attrs_comment = strings::StrCat("Optional attribute setters for ",
op_def.name(), " :\n\n");
string attrs_comment =
strings::StrCat("Optional attribute setters for ", op_name, " :\n\n");
for (int i = 0; i < op_def.attr_size(); ++i) {
const auto& attr(op_def.attr(i));
@ -603,7 +654,13 @@ void OpInfo::WriteClassDecl(WritableFile* h) const {
";\n");
}
strings::StrAppend(&class_decl, "};\n\n");
strings::StrAppend(&class_decl, "};\n");
if (!aliases.empty()) {
for (const auto& alias : aliases) {
strings::StrAppend(&class_decl, "typedef ", op_name, " ", alias, ";\n");
}
}
strings::StrAppend(&class_decl, "\n");
TF_CHECK_OK(h->Append(class_decl));
}
@ -642,7 +699,7 @@ void OpInfo::GetOutput(string* out) const {
for (int i = 0; i < op_def.output_arg_size(); ++i) {
const string arg_range = strings::StrCat(
"_outputs_range[\"", op_def.output_arg(i).name(), "\"]");
"_outputs_range[\"", graph_op_def.output_arg(i).name(), "\"]");
if (is_list_output[i]) {
strings::StrAppend(out, " for (int64 i = ", arg_range, ".first; i < ",
arg_range, ".second; ++i)\n");
@ -673,17 +730,18 @@ string OpInfo::GetConstructorBody() const {
}
strings::StrAppend(&body, " ::tensorflow::Node* ret;\n");
strings::StrAppend(&body, " const auto unique_name = ", scope_str,
".GetUniqueNameForOp(\"", op_def.name(), "\");\n");
strings::StrAppend(&body, " const auto unique_name = ", scope_str,
".GetUniqueNameForOp(\"", op_name, "\");\n");
strings::StrAppend(
&body, " auto builder = ::tensorflow::NodeBuilder(unique_name, \"",
op_def.name(), "\")\n");
graph_op_def.name(), "\")\n");
const string spaces = " ";
for (int i = 0; i < op_def.input_arg_size(); ++i) {
const auto& arg(op_def.input_arg(i));
strings::StrAppend(&body, spaces, ".Input(_", arg.name(), ")\n");
}
for (int i = 0; i < op_def.attr_size(); ++i) {
const auto& graph_attr(graph_op_def.attr(i));
const auto& attr(op_def.attr(i));
if (inferred_input_attrs.find(attr.name()) != inferred_input_attrs.end()) {
continue;
@ -691,7 +749,7 @@ string OpInfo::GetConstructorBody() const {
const string attr_name = attr.has_default_value()
? strings::StrCat("attrs.", attr.name(), "_")
: AvoidCPPKeywords(attr.name());
strings::StrAppend(&body, spaces, ".Attr(\"", attr.name(), "\", ",
strings::StrAppend(&body, spaces, ".Attr(\"", graph_attr.name(), "\", ",
attr_name, ")\n");
}
strings::StrAppend(&body, " ;\n");
@ -736,23 +794,17 @@ void OpInfo::WriteClassDef(WritableFile* cc) const {
TF_CHECK_OK(cc->Append(class_def));
}
void WriteCCOp(const OpDef& op_def, WritableFile* h, WritableFile* cc) {
OpInfo op_info(op_def);
void WriteCCOp(const OpDef& graph_op_def, const OpDef& interface_op_def,
const std::vector<string>& aliases, WritableFile* h,
WritableFile* cc) {
OpInfo op_info(graph_op_def, interface_op_def, aliases);
op_info.WriteClassDecl(h);
op_info.WriteClassDef(cc);
}
} // namespace
void WriteCCOps(const OpList& ops, const std::string& dot_h_fname,
const std::string& dot_cc_fname) {
Env* env = Env::Default();
std::unique_ptr<WritableFile> h = nullptr;
std::unique_ptr<WritableFile> cc = nullptr;
TF_CHECK_OK(env->NewWritableFile(dot_h_fname, &h));
TF_CHECK_OK(env->NewWritableFile(dot_cc_fname, &cc));
void StartFiles(bool internal, const string& dot_h_fname, WritableFile* h,
WritableFile* cc, string* op_header_guard) {
const string header =
R"header(// This file is MACHINE GENERATED! Do not edit.
@ -765,18 +817,22 @@ void WriteCCOps(const OpList& ops, const std::string& dot_h_fname,
)header";
// TODO(keveman): Make namespaces configurable.
const string namespace_begin = R"namespace(
const string namespace_begin = internal ? R"namespace(
namespace tensorflow {
namespace ops {
namespace internal {
// NOTE: This namespace has internal TensorFlow details that
// are not part of TensorFlow's public API.
)namespace"
: R"namespace(
namespace tensorflow {
namespace ops {
)namespace";
const string footer = R"footer(} // namespace ops
} // namespace tensorflow
)footer";
const string op_header = GetPath(dot_h_fname);
const string op_header_guard = ToGuard(op_header);
*op_header_guard = ToGuard(op_header);
const string cc_header = strings::StrCat(
R"include(// This file is MACHINE GENERATED! Do not edit.
@ -788,27 +844,25 @@ namespace ops {
TF_CHECK_OK(h->Append(
strings::StrCat("// This file is MACHINE GENERATED! Do not edit.\n\n"
"#ifndef ",
op_header_guard,
*op_header_guard,
"\n"
"#define ",
op_header_guard, "\n\n")));
*op_header_guard, "\n\n")));
TF_CHECK_OK(h->Append(header));
TF_CHECK_OK(h->Append(namespace_begin));
TF_CHECK_OK(cc->Append(cc_header));
}
for (const auto& op_def : ops.op()) {
// Skip deprecated ops.
// TODO(josh11b): If needed, can put them into a "deprecated" namespace
// instead of skipping.
if (op_def.has_deprecation() &&
op_def.deprecation().version() <= TF_GRAPH_DEF_VERSION) {
continue;
}
// We use a hand-written wrapper for "Const", since the generated
// code depends on it.
if (op_def.name() == "Const") continue;
WriteCCOp(op_def, h.get(), cc.get());
}
void FinishFiles(bool internal, WritableFile* h, WritableFile* cc,
const string& op_header_guard) {
const string footer = internal ? R"footer(} // namespace internal
} // namespace ops
} // namespace tensorflow
)footer"
:
R"footer(} // namespace ops
} // namespace tensorflow
)footer";
TF_CHECK_OK(h->Append(footer));
TF_CHECK_OK(
@ -819,4 +873,82 @@ namespace ops {
TF_CHECK_OK(h->Close());
}
string MakeInternal(const string& fname) {
auto dot_pos = fname.rfind('.');
if (dot_pos == string::npos) {
return strings::StrCat(fname, "_internal");
} else {
return strings::StrCat(fname.substr(0, dot_pos), "_internal",
fname.substr(dot_pos));
}
}
} // namespace
void WriteCCOps(const OpList& ops, const string& dot_h_fname,
const string& dot_cc_fname, const string& overrides_fnames) {
Env* env = Env::Default();
// Load the override map.
OpGenOverrideMap override_map;
if (!overrides_fnames.empty()) {
override_map.LoadFileList(env, overrides_fnames);
}
// Write the initial boilerplate to the .h and .cc files.
std::unique_ptr<WritableFile> h = nullptr;
std::unique_ptr<WritableFile> cc = nullptr;
TF_CHECK_OK(env->NewWritableFile(dot_h_fname, &h));
TF_CHECK_OK(env->NewWritableFile(dot_cc_fname, &cc));
string op_header_guard;
StartFiles(false, dot_h_fname, h.get(), cc.get(), &op_header_guard);
// Create the internal versions of these files for the hidden ops.
std::unique_ptr<WritableFile> internal_h = nullptr;
std::unique_ptr<WritableFile> internal_cc = nullptr;
const string internal_dot_h_fname = MakeInternal(dot_h_fname);
TF_CHECK_OK(env->NewWritableFile(internal_dot_h_fname, &internal_h));
TF_CHECK_OK(env->NewWritableFile(MakeInternal(dot_cc_fname), &internal_cc));
string internal_op_header_guard;
StartFiles(true /* internal */, internal_dot_h_fname, internal_h.get(),
internal_cc.get(), &internal_op_header_guard);
for (const auto& graph_op_def : ops.op()) {
// Skip deprecated ops.
// TODO(josh11b): If needed, can put them into a "deprecated" namespace
// instead of skipping.
if (graph_op_def.has_deprecation() &&
graph_op_def.deprecation().version() <= TF_GRAPH_DEF_VERSION) {
continue;
}
// We use a hand-written wrapper for "Const", since the generated
// code depends on it.
if (graph_op_def.name() == "Const") continue;
// Incorporate overrides from override_map.
OpDef interface_op_def = graph_op_def;
const OpGenOverride* op_override =
override_map.ApplyOverride(&interface_op_def);
std::vector<string> aliases;
if (op_override) {
if (op_override->skip()) continue;
aliases.assign(op_override->alias().begin(), op_override->alias().end());
if (op_override->hide()) {
// Write hidden ops to _internal.h and _internal.cc.
WriteCCOp(graph_op_def, interface_op_def, aliases, internal_h.get(),
internal_cc.get());
continue;
}
}
// This isn't a hidden op, write it to the main files.
WriteCCOp(graph_op_def, interface_op_def, aliases, h.get(), cc.get());
}
FinishFiles(false, h.get(), cc.get(), op_header_guard);
FinishFiles(true /* internal */, internal_h.get(), internal_cc.get(),
internal_op_header_guard);
}
} // namespace tensorflow

5
tensorflow/cc/framework/cc_op_gen.h
View File

@ -17,12 +17,13 @@ limitations under the License.
#define THIRD_PARTY_TENSORFLOW_CC_FRAMEWORK_CC_OP_GEN_H_
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
/// Result is written to files dot_h and dot_cc.
void WriteCCOps(const OpList& ops, const std::string& dot_h_fname,
const std::string& dot_cc_fname);
void WriteCCOps(const OpList& ops, const string& dot_h_fname,
const string& dot_cc_fname, const string& overrides_fnames);
} // namespace tensorflow

15
tensorflow/cc/framework/cc_op_gen_main.cc
View File

@ -24,10 +24,10 @@ namespace tensorflow {
namespace {
void PrintAllCCOps(const std::string& dot_h, const std::string& dot_cc,
bool include_internal) {
const std::string& overrides_fnames, bool include_internal) {
OpList ops;
OpRegistry::Global()->Export(include_internal, &ops);
WriteCCOps(ops, dot_h, dot_cc);
WriteCCOps(ops, dot_h, dot_cc, overrides_fnames);
}
} // namespace
@ -35,15 +35,18 @@ void PrintAllCCOps(const std::string& dot_h, const std::string& dot_cc,
int main(int argc, char* argv[]) {
tensorflow::port::InitMain(argv[0], &argc, &argv);
if (argc != 4) {
if (argc != 5) {
for (int i = 1; i < argc; ++i) {
fprintf(stderr, "Arg %d = %s\n", i, argv[i]);
}
fprintf(stderr,
"Usage: %s out.h out.cc include_internal\n"
"Usage: %s out.h out.cc overrides1.pbtxt,2.pbtxt include_internal\n"
" include_internal: 1 means include internal ops\n",
argv[0]);
exit(1);
}
bool include_internal = tensorflow::StringPiece("1") == argv[3];
tensorflow::PrintAllCCOps(argv[1], argv[2], include_internal);
bool include_internal = tensorflow::StringPiece("1") == argv[4];
tensorflow::PrintAllCCOps(argv[1], argv[2], argv[3], include_internal);
return 0;
}

2
tensorflow/cc/framework/cc_ops_test.cc
View File

@ -68,7 +68,7 @@ TEST(CCOpTest, Attrs) {
TEST(CCOpTest, SplitConcat) {
Scope root = Scope::NewRootScope();
Split p(root, 0, {{1}, {2}}, 2);
auto c = Concat(root, 0, {p[0], p[1]});
auto c = Concat(root, {p[0], p[1]}, 0);
TF_EXPECT_OK(root.status());
Tensor out;
test::GetTensor(root, c, &out);

9
tensorflow/cc/gradients/array_grad.cc
View File

@ -15,6 +15,7 @@ limitations under the License.
#include <vector>
#include "tensorflow/cc/ops/array_ops_internal.h"
#include "tensorflow/cc/ops/standard_ops.h"
#include "tensorflow/cc/framework/grad_op_registry.h"
@ -92,7 +93,7 @@ Status SplitGrad(const Scope& scope, const Operation& op,
const std::vector<Output>& grad_inputs,
std::vector<Output>* grad_outputs) {
grad_outputs->push_back(NoGradient());
grad_outputs->push_back(Concat(scope, op.input(0), grad_inputs));
grad_outputs->push_back(Concat(scope, grad_inputs, op.input(0)));
return scope.status();
}
REGISTER_GRADIENT_OP("Split", SplitGrad);
@ -219,7 +220,7 @@ Status ReverseGrad(const Scope& scope, const Operation& op,
grad_outputs->push_back(NoGradient());
return scope.status();
}
REGISTER_GRADIENT_OP("Reverse", ReverseGrad);
REGISTER_GRADIENT_OP("ReverseV2", ReverseGrad);
Status ScatterNdGrad(const Scope& scope, const Operation& op,
const std::vector<Output>& grad_inputs,
@ -319,8 +320,8 @@ Status MirrorPadGrad(const Scope& scope, const Operation& op,
std::vector<Output>* grad_outputs) {
string mode;
TF_RETURN_IF_ERROR(GetNodeAttr(op.node()->def(), "mode", &mode));
grad_outputs->push_back(
tensorflow::ops::MirrorPadGrad(scope, grad_inputs[0], op.input(1), mode));
grad_outputs->push_back(tensorflow::ops::internal::MirrorPadGrad(
scope, grad_inputs[0], op.input(1), mode));
grad_outputs->push_back(NoGradient());
return scope.status();
}

5
tensorflow/cc/gradients/array_grad_test.cc
View File

@ -17,12 +17,14 @@ limitations under the License.
#include "tensorflow/cc/framework/gradient_checker.h"
#include "tensorflow/cc/framework/testutil.h"
#include "tensorflow/cc/gradients/grad_testutil.h"
#include "tensorflow/cc/ops/array_ops_internal.h"
#include "tensorflow/cc/ops/standard_ops.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/lib/core/status_test_util.h"
namespace tensorflow {
using namespace ops; // NOLINT(build/namespaces)
using ops::internal::MirrorPadGrad;
namespace {
@ -207,8 +209,7 @@ TEST_F(ArrayGradTest, ReverseSequenceGrad) {
TEST_F(ArrayGradTest, ReverseGrad) {
TensorShape shape({5, 2, 5});
auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
auto reverse_dims = Const(scope_, {true, false, true});
auto y = Reverse(scope_, x, reverse_dims);
auto y = Reverse(scope_, x, {0, 2});
RunTest(x, shape, y, shape);
}

42
tensorflow/cc/ops/op_gen_overrides.pbtxt Normal file
View File

@ -0,0 +1,42 @@
# array_ops
op { name: "BroadcastArgs" rename_to: "BroadcastDynamicShape" }
op { name: "ConcatOffset" skip: true } # Maybe should just be hidden?
op { name: "Concat" skip: true }
op { name: "ConcatV2" rename_to: "Concat" }
op { name: "MirrorPadGrad" hide: true }
op { name: "Reverse" skip: true }
op { name: "ReverseV2" rename_to: "Reverse" }
# candidate_sampling_ops
# control_flow_ops
# ctc_ops
# data_flow_ops
op { name: "FakeQueue" skip: true }
# functional_ops
# image_ops
# io_ops
# linalg_ops
# logging_ops
# math_ops
op { name: "All" alias: "ReduceAll" input_rename: { from: "reduction_indices" to: "axis" } }
op { name: "Any" alias: "ReduceAny" input_rename: { from: "reduction_indices" to: "axis" } }
op { name: "Max" alias: "ReduceMax" input_rename: { from: "reduction_indices" to: "axis" } }
op { name: "Mean" alias: "ReduceMean" input_rename: { from: "reduction_indices" to: "axis" } }
op { name: "Min" alias: "ReduceMin" input_rename: { from: "reduction_indices" to: "axis" } }
op { name: "Prod" alias: "ReduceProd" input_rename: { from: "reduction_indices" to: "axis" } }
op { name: "Sum" alias: "ReduceSum" input_rename: { from: "reduction_indices" to: "axis" } }
# nn_ops
op { name: "TopKV2" rename_to: "TopK" alias: "TopKV2" } # TODO(josh11b): delete "TopKV2" alias once users updated
# parsing_ops
# random_ops
# script_ops
# sdca_ops
# state_ops
# sparse_ops
# string_ops
# user_ops
# training_ops
# word2vec deprecated ops

1
tensorflow/contrib/makefile/Makefile
View File

@ -454,6 +454,7 @@ $(wildcard tensorflow/core/*/*/*testutil*) \
$(wildcard tensorflow/core/*/*/*testlib*) \
$(wildcard tensorflow/core/*/*/*main.cc) \
$(wildcard tensorflow/core/debug/*.cc) \
$(wildcard tensorflow/core/framework/op_gen_lib.cc) \
$(wildcard tensorflow/core/graph/dot.*) \
$(wildcard tensorflow/core/lib/gif/*) \
$(wildcard tensorflow/core/lib/io/zlib*) \

22
tensorflow/core/BUILD
View File

@ -328,7 +328,6 @@ tf_cuda_library(
"framework/op.h",
"framework/op_def_builder.h",
"framework/op_def_util.h",
"framework/op_gen_lib.h",
"framework/op_kernel.h",
"framework/partial_tensor_shape.h",
"framework/queue_interface.h",
@ -384,6 +383,25 @@ tf_cuda_library(
deps = [":framework_internal"],
)
tf_proto_library_cc(
name = "op_gen_overrides_proto",
srcs = ["framework/op_gen_overrides.proto"],
cc_api_version = 2,
protodeps = [":protos_all"],
visibility = ["//visibility:public"],
)
cc_library(
name = "op_gen_lib",
srcs = ["framework/op_gen_lib.cc"],
hdrs = ["framework/op_gen_lib.h"],
visibility = ["//visibility:public"],
deps = [
":lib",
":op_gen_overrides_proto_cc",
],
)
cc_library(
name = "framework_lite",
srcs = tf_additional_minimal_lib_srcs(),
@ -761,6 +779,7 @@ filegroup(
"**/*testlib*",
"**/*main.cc",
"debug/**/*",
"framework/op_gen_*",
"graph/dot.*",
"lib/jpeg/**/*",
"lib/png/**/*",
@ -1250,6 +1269,7 @@ tf_cuda_library(
"util/reporter.h",
"util/reporter.cc",
"framework/fake_input.*",
"framework/op_gen_lib.*",
"util/memmapped_file_system.*",
"util/memmapped_file_system_writer.*",
"util/version_info.cc",

14
tensorflow/core/common_runtime/shape_refiner.cc
View File

@ -397,18 +397,22 @@ Status ShapeRefiner::ConstantPartialShape(InferenceContext* target_context,
}
}
*result = target_context->MakeShape(dims);
} else if (src_op == "Concat") {
} else if (src_op == "Concat" || src_op == "ConcatV2") {
*result = target_context->Scalar();
// For Concat, input 0 is concat dim; for V2 it is the last input.
const int concat_dim =
src_op == "Concat" ? 0 : src_context->num_inputs() - 1;
// Concat is concatenating its input shape vectors.
// input 0 is ignored as it is the concat dim and will always be 0.
for (int i = 1; i < src_context->num_inputs(); ++i) {
for (int i = 0; i < src_context->num_inputs(); ++i) {
// Concat dim is ignored (and will always be a scalar).
if (i == concat_dim) continue;
ShapeHandle sub_result;
TF_RETURN_IF_ERROR(ConstantPartialShape(target_context, input_edge->src(),
i, &sub_result));
if (!target_context->RankKnown(sub_result)) {
// Failed to evaluate. Treat the output as completely unknown.
// TODO(cwhipkey): we could rely on all inputs being the same size, so
// figure that size out and append the right number of unknown dims.
// TODO(cwhipkey): we could rely on all inputs being the same rank, so
// figure that rank out and append the right number of unknown dims.
*result = target_context->UnknownShape();
return Status::OK();
}

6
tensorflow/core/common_runtime/shape_refiner_test.cc
View File

@ -642,7 +642,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_Concat) {
const_input,
}; // clang-format on
auto concat_dim = ops::Const(root, 0);
auto concat = ops::Concat(root, concat_dim, concat_inputs);
auto concat = ops::Concat(root, concat_inputs, concat_dim);
TF_ASSERT_OK(root.status());
Node* result;
@ -684,7 +684,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_ConcatWithUnknown) {
ops::Shape(root, Output(unknown)),
}; // clang-format on
auto concat_dim = ops::Const(root, 0);
auto concat = ops::Concat(root, concat_dim, concat_inputs);
auto concat = ops::Concat(root, concat_inputs, concat_dim);
TF_ASSERT_OK(root.status());
Node* result;
@ -726,7 +726,7 @@ TEST(ShapeRefinerTest, ConstantValueAsShape_ConcatInvalidDimValue) {
const_input,
}; // clang-format on
auto concat_dim = ops::Const(root, 0);
auto concat = ops::Concat(root, concat_dim, concat_inputs);
auto concat = ops::Concat(root, concat_inputs, concat_dim);
TF_ASSERT_OK(root.status());
Node* result;

135
tensorflow/core/framework/op_gen_lib.cc
View File

@ -15,6 +15,9 @@ limitations under the License.
#include "tensorflow/core/framework/op_gen_lib.h"
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
namespace tensorflow {
@ -67,4 +70,136 @@ bool ConsumeEquals(StringPiece* description) {
return false;
}
Status OpGenOverrideMap::LoadFileList(Env* env, const string& filenames) {
std::vector<string> v = str_util::Split(filenames, ",");
for (const string& f : v) {
TF_RETURN_IF_ERROR(LoadFile(env, f));
}
return Status::OK();
}
Status OpGenOverrideMap::LoadFile(Env* env, const string& filename) {
if (filename.empty()) return Status::OK();
string contents;
TF_RETURN_IF_ERROR(ReadFileToString(env, filename, &contents));
OpGenOverrides all;
protobuf::TextFormat::ParseFromString(contents, &all);
for (const auto& one : all.op()) {
map_[one.name()] = one;
}
return Status::OK();
}
static void StringReplace(const string& from, const string& to, string* s) {
std::vector<string> rest = str_util::Split(*s, from);
*s = str_util::Join(rest, to.c_str());
}
static void RenameInDocs(const string& from, const string& to, OpDef* op_def) {
const string from_quoted = strings::StrCat("`", from, "`");
const string to_quoted = strings::StrCat("`", to, "`");
for (int i = 0; i < op_def->input_arg_size(); ++i) {
if (!op_def->input_arg(i).description().empty()) {
StringReplace(from_quoted, to_quoted,
op_def->mutable_input_arg(i)->mutable_description());
}
}
for (int i = 0; i < op_def->output_arg_size(); ++i) {
if (!op_def->output_arg(i).description().empty()) {
StringReplace(from_quoted, to_quoted,
op_def->mutable_output_arg(i)->mutable_description());
}
}
for (int i = 0; i < op_def->attr_size(); ++i) {
if (!op_def->attr(i).description().empty()) {
StringReplace(from_quoted, to_quoted,
op_def->mutable_attr(i)->mutable_description());
}
}
if (!op_def->summary().empty()) {
StringReplace(from_quoted, to_quoted, op_def->mutable_summary());
}
if (!op_def->description().empty()) {
StringReplace(from_quoted, to_quoted, op_def->mutable_summary());
}
}
const OpGenOverride* OpGenOverrideMap::ApplyOverride(OpDef* op_def) const {
// Look up
const auto iter = map_.find(op_def->name());
if (iter == map_.end()) return nullptr;
const OpGenOverride& proto = iter->second;
// Apply overrides from `proto`.
if (!proto.rename_to().empty()) {
op_def->set_name(proto.rename_to());
RenameInDocs(proto.name(), proto.rename_to(), op_def);
}
for (const auto& attr_default : proto.attr_default()) {
bool found = false;
for (int i = 0; i < op_def->attr_size(); ++i) {
if (op_def->attr(i).name() == attr_default.name()) {
*op_def->mutable_attr(i)->mutable_default_value() =
attr_default.value();
found = true;
break;
}
}
if (!found) {
LOG(WARNING) << proto.name() << " can't find attr " << attr_default.name()
<< " to override default";
}
}
for (const auto& attr_rename : proto.attr_rename()) {
bool found = false;
for (int i = 0; i < op_def->attr_size(); ++i) {
if (op_def->attr(i).name() == attr_rename.from()) {
*op_def->mutable_attr(i)->mutable_name() = attr_rename.to();
found = true;
break;
}
}
if (found) {
RenameInDocs(attr_rename.from(), attr_rename.to(), op_def);
} else {
LOG(WARNING) << proto.name() << " can't find attr " << attr_rename.from()
<< " to rename";
}
}
for (const auto& input_rename : proto.input_rename()) {
bool found = false;
for (int i = 0; i < op_def->input_arg_size(); ++i) {
if (op_def->input_arg(i).name() == input_rename.from()) {
*op_def->mutable_input_arg(i)->mutable_name() = input_rename.to();
found = true;
break;
}
}
if (found) {
RenameInDocs(input_rename.from(), input_rename.to(), op_def);
} else {
LOG(WARNING) << proto.name() << " can't find input "
<< input_rename.from() << " to rename";
}
}
for (const auto& output_rename : proto.output_rename()) {
bool found = false;
for (int i = 0; i < op_def->output_arg_size(); ++i) {
if (op_def->output_arg(i).name() == output_rename.from()) {
*op_def->mutable_output_arg(i)->mutable_name() = output_rename.to();
found = true;
break;
}
}
if (found) {
RenameInDocs(output_rename.from(), output_rename.to(), op_def);
} else {
LOG(WARNING) << proto.name() << " can't find output "
<< output_rename.from() << " to rename";
}
}
return &proto;
}
} // namespace tensorflow

30
tensorflow/core/framework/op_gen_lib.h
View File

@ -17,7 +17,12 @@ limitations under the License.
#define TENSORFLOW_FRAMEWORK_OP_GEN_LIB_H_
#include <string>
#include <unordered_map>
#include "tensorflow/core/framework/op_def.pb.h"
#include "tensorflow/core/framework/op_gen_overrides.pb.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/env.h"
namespace tensorflow {
@ -34,6 +39,31 @@ string WordWrap(StringPiece prefix, StringPiece str, int width);
// returns false.
bool ConsumeEquals(StringPiece* description);
// Takes a list of files with OpGenOverrides text protos, and allows you to
// look up the specific override for any given op.
class OpGenOverrideMap {
public:
// `filenames` is a comma-separated list of file names. If an op
// is mentioned in more than one file, the last one takes priority.
Status LoadFileList(Env* env, const string& filenames);
// Load a single file. If more than one file is loaded, later ones
// take priority for any ops in common.
Status LoadFile(Env* env, const string& filename);
// Look up the override for `*op_def` from the loaded files, and
// mutate `*op_def` to reflect the requested changes. Does not apply
// 'skip', 'hide', or 'alias' overrides. Caller has to deal with
// those since they can't be simulated by mutating `*op_def`.
// Returns nullptr if op is not in any loaded file. Otherwise, the
// pointer must not be referenced beyond the lifetime of *this or
// the next file load.
const OpGenOverride* ApplyOverride(OpDef* op_def) const;
private:
std::unordered_map<string, OpGenOverride> map_;
};
} // namespace tensorflow
#endif // TENSORFLOW_FRAMEWORK_OP_GEN_LIB_H_

67
tensorflow/core/framework/op_gen_overrides.proto Normal file
View File

@ -0,0 +1,67 @@
// Defines the text format for adding per-op overrides for client
// language op code generators.
syntax = "proto3";
package tensorflow;
import "tensorflow/core/framework/attr_value.proto";
// Used to override the default API & behavior in the generated code
// for client languages, from what you would get from the OpDef alone.
// This is so we can evolve the API while remaining backwards
// compatible when interpretting old graphs. Overrides go in an
// "op_gen_overrides.pbtxt" file with a text-format OpGenOverrides
// message. Right now these only apply to the C++ API.
// TODO(josh11b): In the future there will be a common set of overrides
// and per-client-language overrides.
//
// WARNING: Be *very* careful using these features -- these overrides
// can change the semantics of existing code. These changes may need
// to wait until a major release of TensorFlow to avoid breaking our
// compatibility promises.
message OpGenOverride {
// Name of the op to apply overrides to.
string name = 1;
// Do not include this op in the generated API.
// If `skip` is true, all other overrides are ignored for this op.
bool skip = 2;
// Hide this op by putting it into an internal namespace (or whatever
// is appropriate in the target language).
bool hide = 3;
// Use a different name in the API than the op's name. Note that
// the op's name in `backticks` will also be replaced in the docs.
string rename_to = 4;
// Create *additional* API endpoints with different names (contrast
// with rename_to, which affects the original name).
repeated string alias = 5;
// Map the name of an attr to a new default value to use. This
// default will be used when creating new graphs, as opposed to the
// default in the OpDef, which will be used when interpreting old
// GraphDefs. If this attr is also renamed (using attr_rename
// below), use the original name of the attr.
message AttrDefault {
string name = 1;
AttrValue value = 2;
}
repeated AttrDefault attr_default = 6;
// Change the name used to access attrs/inputs/outputs in the API
// from what is used in the GraphDef. Note that these names in
// `backticks` will also be replaced in the docs.
message Rename {
string from = 1;
string to = 2;
}
repeated Rename attr_rename = 7;
repeated Rename input_rename = 8;
repeated Rename output_rename = 9;
}
message OpGenOverrides {
repeated OpGenOverride op = 1;
}

3
tensorflow/core/ops/nn_ops.cc
View File

@ -1860,6 +1860,7 @@ values: The `k` largest elements along each last dimensional slice.
indices: The indices of `values` within the last dimension of `input`.
)doc");
// This is the same as `TopK`, but takes `k` as in input rather than an attr.
REGISTER_OP("TopKV2")
.Input("input: T")
.Input("k: int32")
@ -1882,8 +1883,6 @@ row (resp. vector along the last dimension). Thus,
If two elements are equal, the lower-index element appears first.
This is the same as `TopK`, but takes `k` as in input rather than an attr.
input: 1-D or higher with last dimension at least `k`.
k: 0-D. Number of top elements to look for along the last dimension (along each
row for matrices).

2
tensorflow/examples/label_image/main.cc
View File

@ -163,7 +163,7 @@ Status GetTopLabels(const std::vector<Tensor>& outputs, int how_many_labels,
using namespace ::tensorflow::ops; // NOLINT(build/namespaces)
string output_name = "top_k";
TopKV2(root.WithOpName(output_name), outputs[0], how_many_labels);
TopK(root.WithOpName(output_name), outputs[0], how_many_labels);
// This runs the GraphDef network definition that we've just constructed, and
// returns the results in the output tensors.
tensorflow::GraphDef graph;

2
tensorflow/examples/multibox_detector/main.cc
View File

@ -184,7 +184,7 @@ Status GetTopDetections(const std::vector<Tensor>& outputs, int how_many_labels,
using namespace ::tensorflow::ops; // NOLINT(build/namespaces)
string output_name = "top_k";
TopKV2(root.WithOpName(output_name), outputs[0], how_many_labels);
TopK(root.WithOpName(output_name), outputs[0], how_many_labels);
// This runs the GraphDef network definition that we've just constructed, and
// returns the results in the output tensors.
tensorflow::GraphDef graph;

1
tensorflow/python/BUILD
View File

@ -315,6 +315,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
"//tensorflow/core:framework",
"//tensorflow/core:op_gen_lib",
"//tensorflow/core:protos_cc",
],
alwayslink = 1,

44
tensorflow/tensorflow.bzl
View File

@ -143,6 +143,7 @@ def tf_gen_op_libs(op_lib_names, deps=None):
def tf_gen_op_wrapper_cc(name, out_ops_file, pkg="",
op_gen="//tensorflow/cc:cc_op_gen_main",
deps=None,
override_file=None,
include_internal_ops=0):
# Construct an op generator binary for these ops.
tool = out_ops_file + "_gen_cc"
@ -156,12 +157,21 @@ def tf_gen_op_wrapper_cc(name, out_ops_file, pkg="",
deps = [op_gen] + deps
)
if override_file == None:
srcs = []
override_arg = ","
else:
srcs = [override_file]
override_arg = "$(location " + override_file + ")"
native.genrule(
name=name + "_genrule",
outs=[out_ops_file + ".h", out_ops_file + ".cc"],
outs=[out_ops_file + ".h", out_ops_file + ".cc",
out_ops_file + "_internal.h", out_ops_file + "_internal.cc"],
srcs=srcs,
tools=[":" + tool],
cmd=("$(location :" + tool + ") $(location :" + out_ops_file + ".h) " +
"$(location :" + out_ops_file + ".cc) " + str(include_internal_ops)))
"$(location :" + out_ops_file + ".cc) " + override_arg + " " +
str(include_internal_ops)))
# Given a list of "op_lib_names" (a list of files in the ops directory
# without their .cc extensions), generate individual C++ .cc and .h
@ -181,6 +191,15 @@ def tf_gen_op_wrapper_cc(name, out_ops_file, pkg="",
# hdrs = [ "ops/array_ops.h",
# "ops/math_ops.h" ],
# deps = [ ... ])
#
# Plus a private library for the "hidden" ops.
# cc_library(name = "tf_ops_lib_internal",
# srcs = [ "ops/array_ops_internal.cc",
# "ops/math_ops_internal.cc" ],
# hdrs = [ "ops/array_ops_internal.h",
# "ops/math_ops_internal.h" ],
# deps = [ ... ])
# TODO(josh11b): Cleaner approach for hidden ops.
def tf_gen_op_wrappers_cc(name,
op_lib_names=[],
other_srcs=[],
@ -192,16 +211,21 @@ def tf_gen_op_wrappers_cc(name,
"//tensorflow/cc:const_op",
],
op_gen="//tensorflow/cc:cc_op_gen_main",
override_file=None,
include_internal_ops=0,
visibility=None):
subsrcs = other_srcs
subhdrs = other_hdrs
internalsrcs = []
internalhdrs = []
for n in op_lib_names:
tf_gen_op_wrapper_cc(
n, "ops/" + n, pkg=pkg, op_gen=op_gen,
n, "ops/" + n, pkg=pkg, op_gen=op_gen, override_file=override_file,
include_internal_ops=include_internal_ops)
subsrcs += ["ops/" + n + ".cc"]
subhdrs += ["ops/" + n + ".h"]
internalsrcs += ["ops/" + n + "_internal.cc"]
internalhdrs += ["ops/" + n + "_internal.h"]
native.cc_library(name=name,
srcs=subsrcs,
@ -217,6 +241,20 @@ def tf_gen_op_wrappers_cc(name,
copts=tf_copts(),
alwayslink=1,
visibility=visibility)
native.cc_library(name=name + "_internal",
srcs=internalsrcs,
hdrs=internalhdrs,
deps=deps + if_not_android([
"//tensorflow/core:core_cpu",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
]) + if_android([
"//tensorflow/core:android_tensorflow_lib",
]),
copts=tf_copts(),
alwayslink=1,
visibility=["//visibility:private"])
# Invoke this rule in .../tensorflow/python to build the wrapper library.
def tf_gen_op_wrapper_py(name, out=None, hidden=None, visibility=None, deps=[],

2
tensorflow/tools/graph_transforms/quantize_nodes_test.cc
View File

@ -296,7 +296,7 @@ class QuantizeNodesTest : public ::testing::Test {
Output b_op = Const(root.WithOpName("b_op"), Input::Initializer(b_tensor));
Output concat_op =
Concat(root.WithOpName("concat_op"), shape_op, {a_op, b_op});
Concat(root.WithOpName("concat_op"), {a_op, b_op}, shape_op);
GraphDef float_graph_def;
TF_ASSERT_OK(root.ToGraphDef(&float_graph_def));