[XLA] Add expander pass for kLogistic.
Adding an HLO pass that expands the kLogistic HLO into a desired sequence of different HLOs. Currently two different strategies are added. 1) A lowering through an expansion using TAHN (0.5 + 0.5 * tanh(0.5 * x)) 2) A lowering through an expansion using EXP (1.0 / (1.0 + exp(-x))) PiperOrigin-RevId: 318208462 Change-Id: Ibcfba8e95f76c85cdbffc42566f5cec5e663c72b
This commit is contained in:
Marcello Maggioni
TensorFlower Gardener
parent
fdc80210a7
commit
09d0ef73ff
tensorflow/compiler/xla/service
@ -2044,6 +2044,73 @@ tf_cc_test(
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "logistic_expander",
|
||||
srcs = ["logistic_expander.cc"],
|
||||
hdrs = ["logistic_expander.h"],
|
||||
deps = [
|
||||
":hlo",
|
||||
":hlo_casting_utils",
|
||||
":hlo_creation_utils",
|
||||
":hlo_evaluator",
|
||||
":hlo_pass",
|
||||
":hlo_query",
|
||||
":op_expander_pass",
|
||||
":pattern_matcher",
|
||||
"//tensorflow/compiler/xla:comparison_util",
|
||||
"//tensorflow/compiler/xla:literal",
|
||||
"//tensorflow/compiler/xla:literal_util",
|
||||
"//tensorflow/compiler/xla:shape_util",
|
||||
"//tensorflow/compiler/xla:status_macros",
|
||||
"//tensorflow/compiler/xla:types",
|
||||
"//tensorflow/compiler/xla:util",
|
||||
"//tensorflow/compiler/xla:window_util",
|
||||
"//tensorflow/compiler/xla:xla_data_proto_cc",
|
||||
"//tensorflow/core:lib",
|
||||
"//tensorflow/core/platform:logging",
|
||||
"//tensorflow/core/platform:types",
|
||||
"//tensorflow/stream_executor/lib",
|
||||
"@com_google_absl//absl/algorithm:container",
|
||||
"@com_google_absl//absl/container:flat_hash_map",
|
||||
"@com_google_absl//absl/container:flat_hash_set",
|
||||
"@com_google_absl//absl/container:inlined_vector",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/types:optional",
|
||||
"@com_google_absl//absl/types:span",
|
||||
],
|
||||
)
|
||||
|
||||
tf_cc_test(
|
||||
name = "logistic_expander_test",
|
||||
srcs = ["logistic_expander_test.cc"],
|
||||
deps = [
|
||||
":hlo",
|
||||
":hlo_casting_utils",
|
||||
":hlo_creation_utils",
|
||||
":hlo_parser",
|
||||
":hlo_pass",
|
||||
":hlo_pass_pipeline",
|
||||
":logistic_expander",
|
||||
":pattern_matcher",
|
||||
":pattern_matcher_gmock",
|
||||
":shape_inference",
|
||||
"//tensorflow/compiler/xla:literal",
|
||||
"//tensorflow/compiler/xla:shape_util",
|
||||
"//tensorflow/compiler/xla:test",
|
||||
"//tensorflow/compiler/xla:types",
|
||||
"//tensorflow/compiler/xla:util",
|
||||
"//tensorflow/compiler/xla:window_util",
|
||||
"//tensorflow/compiler/xla:xla_data_proto_cc",
|
||||
"//tensorflow/compiler/xla/tests:hlo_test_base",
|
||||
"//tensorflow/compiler/xla/tests:xla_internal_test_main", # fixdeps: keep
|
||||
"//tensorflow/core:lib",
|
||||
"//tensorflow/core:test",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "all_reduce_combiner",
|
||||
srcs = ["all_reduce_combiner.cc"],
|
||||
|
||||
88
tensorflow/compiler/xla/service/logistic_expander.cc
Normal file
88
tensorflow/compiler/xla/service/logistic_expander.cc
Normal file
@ -0,0 +1,88 @@
|
||||
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include "tensorflow/compiler/xla/service/logistic_expander.h"
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "tensorflow/compiler/xla/literal.h"
|
||||
#include "tensorflow/compiler/xla/literal_util.h"
|
||||
#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_computation.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_instruction.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_opcode.h"
|
||||
#include "tensorflow/compiler/xla/shape_util.h"
|
||||
#include "tensorflow/compiler/xla/status_macros.h"
|
||||
#include "tensorflow/compiler/xla/types.h"
|
||||
#include "tensorflow/compiler/xla/util.h"
|
||||
#include "tensorflow/compiler/xla/xla_data.pb.h"
|
||||
#include "tensorflow/core/lib/core/errors.h"
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace xla {
|
||||
|
||||
namespace {
|
||||
|
||||
HloInstruction* ExpandLogisticWithTanh(HloInstruction* logistic) {
|
||||
HloInstruction* operand = logistic->mutable_operand(0);
|
||||
const Shape operand_shape = operand->shape();
|
||||
HloInstruction* half_constant = MakeScalarLike(operand, 0.5f);
|
||||
HloInstruction* tanh_instr =
|
||||
MakeUnaryHlo(HloOpcode::kTanh,
|
||||
MakeBinaryHlo(HloOpcode::kMultiply, half_constant, operand)
|
||||
.ValueOrDie())
|
||||
.ValueOrDie();
|
||||
return MakeBinaryHlo(
|
||||
HloOpcode::kAdd, half_constant,
|
||||
MakeBinaryHlo(HloOpcode::kMultiply, half_constant, tanh_instr)
|
||||
.ValueOrDie())
|
||||
.ValueOrDie();
|
||||
}
|
||||
|
||||
HloInstruction* ExpandLogisticWithExp(HloInstruction* logistic) {
|
||||
HloInstruction* operand = logistic->mutable_operand(0);
|
||||
const Shape operand_shape = operand->shape();
|
||||
// Computing 1.0 / (1.0 - exp(-x))
|
||||
HloInstruction* one_constant = MakeScalarLike(operand, 1.0f);
|
||||
HloInstruction* exp_instr =
|
||||
MakeUnaryHlo(HloOpcode::kExp,
|
||||
MakeUnaryHlo(HloOpcode::kNegate, operand).ValueOrDie())
|
||||
.ValueOrDie();
|
||||
HloInstruction* denominator =
|
||||
MakeBinaryHlo(HloOpcode::kAdd, one_constant, exp_instr).ValueOrDie();
|
||||
return MakeBinaryHlo(HloOpcode::kDivide, one_constant, denominator)
|
||||
.ValueOrDie();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool LogisticExpander::InstructionMatchesPattern(HloInstruction* instruction) {
|
||||
return instruction->opcode() == HloOpcode::kLogistic;
|
||||
}
|
||||
|
||||
StatusOr<HloInstruction*> LogisticExpander::ExpandInstruction(
|
||||
HloInstruction* instruction) {
|
||||
switch (expansion_type_) {
|
||||
case LogisticExpansionType::kTanh:
|
||||
return ExpandLogisticWithTanh(instruction);
|
||||
case LogisticExpansionType::kExp:
|
||||
return ExpandLogisticWithExp(instruction);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace xla
|
||||
53
tensorflow/compiler/xla/service/logistic_expander.h
Normal file
53
tensorflow/compiler/xla/service/logistic_expander.h
Normal file
@ -0,0 +1,53 @@
|
||||
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LOGISTIC_EXPANDER_H_
|
||||
#define TENSORFLOW_COMPILER_XLA_SERVICE_LOGISTIC_EXPANDER_H_
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "tensorflow/compiler/xla/service/hlo_module.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
|
||||
#include "tensorflow/compiler/xla/service/op_expander_pass.h"
|
||||
|
||||
namespace xla {
|
||||
|
||||
enum class LogisticExpansionType {
|
||||
kTanh, // Expands as 0.5 + 0.5*tanh(0.5*x)
|
||||
kExp, // Expands as 1.0 / (1.0 + exp(-x))
|
||||
};
|
||||
|
||||
// A pass which performs expansion of the logistic function.
|
||||
class LogisticExpander : public OpExpanderPass {
|
||||
public:
|
||||
explicit LogisticExpander(LogisticExpansionType expansion_type)
|
||||
: expansion_type_(expansion_type) {}
|
||||
~LogisticExpander() override = default;
|
||||
absl::string_view name() const override { return "logistic-expander"; }
|
||||
|
||||
private:
|
||||
// Returns `true` if `instruction` should be expanded by this pass.
|
||||
bool InstructionMatchesPattern(HloInstruction* instruction) override;
|
||||
// Returns a replacement for `instruction`, or nullptr if no replacement is
|
||||
// needed (e.g. only the to_apply subcomputation of the instruction was
|
||||
// modified).
|
||||
StatusOr<HloInstruction*> ExpandInstruction(
|
||||
HloInstruction* instruction) override;
|
||||
LogisticExpansionType expansion_type_;
|
||||
};
|
||||
|
||||
} // namespace xla
|
||||
|
||||
#endif // TENSORFLOW_COMPILER_XLA_SERVICE_LOGISTIC_EXPANDER_H_
|
||||
103
tensorflow/compiler/xla/service/logistic_expander_test.cc
Normal file
103
tensorflow/compiler/xla/service/logistic_expander_test.cc
Normal file
@ -0,0 +1,103 @@
|
||||
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include "tensorflow/compiler/xla/service/logistic_expander.h"
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/strings/str_join.h"
|
||||
#include "tensorflow/compiler/xla/layout_util.h"
|
||||
#include "tensorflow/compiler/xla/literal.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_computation.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_creation_utils.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_instruction.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_instructions.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_opcode.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_parser.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
|
||||
#include "tensorflow/compiler/xla/service/hlo_pass_pipeline.h"
|
||||
#include "tensorflow/compiler/xla/service/pattern_matcher.h"
|
||||
#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
|
||||
#include "tensorflow/compiler/xla/service/shape_inference.h"
|
||||
#include "tensorflow/compiler/xla/shape_util.h"
|
||||
#include "tensorflow/compiler/xla/test.h"
|
||||
#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
|
||||
#include "tensorflow/compiler/xla/types.h"
|
||||
#include "tensorflow/compiler/xla/window_util.h"
|
||||
#include "tensorflow/compiler/xla/xla_data.pb.h"
|
||||
#include "tensorflow/core/lib/core/status_test_util.h"
|
||||
|
||||
namespace xla {
|
||||
namespace {
|
||||
|
||||
namespace m = match;
|
||||
|
||||
class LogisticExpanderTest : public HloTestBase {};
|
||||
|
||||
// Test that we expand kLogistic with 0.5 + 0.5 * tanh(0.5*x) when the proper
|
||||
// option is enabled.
|
||||
TEST_F(LogisticExpanderTest, ExpandWithTanh) {
|
||||
const char* kModuleStr = R"(
|
||||
HloModule m
|
||||
test {
|
||||
p = f32[2,3] parameter(0)
|
||||
ROOT r = f32[2,3] logistic(p)
|
||||
}
|
||||
)";
|
||||
TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
|
||||
|
||||
auto computation = m->entry_computation();
|
||||
HloInstruction* root = computation->root_instruction();
|
||||
EXPECT_EQ(root->opcode(), HloOpcode::kLogistic);
|
||||
LogisticExpander logistic_expander(LogisticExpansionType::kTanh);
|
||||
ASSERT_TRUE(logistic_expander.Run(m.get()).ValueOrDie());
|
||||
root = computation->root_instruction();
|
||||
EXPECT_THAT(m->entry_computation()->root_instruction(),
|
||||
GmockMatch(m::AddAnyOrder(
|
||||
m::MultiplyAnyOrder(m::Broadcast(m::ConstantScalar(0.5)),
|
||||
m::Tanh(m::MultiplyAnyOrder(
|
||||
m::Broadcast(m::ConstantScalar(0.5)),
|
||||
m::Parameter(0)))),
|
||||
m::Broadcast(m::ConstantScalar(0.5)))));
|
||||
}
|
||||
|
||||
// Test that we expand kLogistic with 1.0 / (1.0 + exp(-x)) when the proper
|
||||
// option is enabled.
|
||||
TEST_F(LogisticExpanderTest, ExpandWithEXP) {
|
||||
const char* kModuleStr = R"(
|
||||
HloModule m
|
||||
test {
|
||||
p = f32[2,3] parameter(0)
|
||||
ROOT r = f32[2,3] logistic(p)
|
||||
}
|
||||
)";
|
||||
TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
|
||||
|
||||
auto computation = m->entry_computation();
|
||||
HloInstruction* root = computation->root_instruction();
|
||||
EXPECT_EQ(root->opcode(), HloOpcode::kLogistic);
|
||||
LogisticExpander logistic_expander(LogisticExpansionType::kExp);
|
||||
ASSERT_TRUE(logistic_expander.Run(m.get()).ValueOrDie());
|
||||
root = computation->root_instruction();
|
||||
EXPECT_THAT(m->entry_computation()->root_instruction(),
|
||||
GmockMatch(m::Divide(
|
||||
m::Broadcast(m::ConstantScalar(1.0)),
|
||||
m::AddAnyOrder(m::Broadcast(m::ConstantScalar(1.0)),
|
||||
m::Exp(m::Negate(m::Parameter(0)))))));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace xla
|
||||
Loading…
Reference in New Issue
Block a user