From 3299b778c3f987de4a05820f82d5b63d35d032bf Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Tue, 10 Mar 2020 11:14:23 -0700 Subject: [PATCH] Add activations to frozen_keras which will be needed by layers. PiperOrigin-RevId: 300134428 Change-Id: I314f38d53afd30b4536b2dcb6d0a02a6d11cd1cb --- tensorflow/python/frozen_keras/BUILD | 12 + tensorflow/python/frozen_keras/activations.py | 453 ++++++++++++++++++ 2 files changed, 465 insertions(+) create mode 100644 tensorflow/python/frozen_keras/activations.py diff --git a/tensorflow/python/frozen_keras/BUILD b/tensorflow/python/frozen_keras/BUILD index 2a41f7485ac..ca01753ca3e 100644 --- a/tensorflow/python/frozen_keras/BUILD +++ b/tensorflow/python/frozen_keras/BUILD @@ -19,6 +19,18 @@ py_library( ], ) +py_library( + name = "activations", + srcs = ["activations.py"], + deps = [ + ":backend", + "//tensorflow/python:math_ops", + "//tensorflow/python:nn", + "//tensorflow/python/frozen_keras/utils:generic_utils", + "@six_archive//:six", + ], +) + py_library( name = "backend", srcs = ["backend.py"], diff --git a/tensorflow/python/frozen_keras/activations.py b/tensorflow/python/frozen_keras/activations.py new file mode 100644 index 00000000000..70dfa1a826b --- /dev/null +++ b/tensorflow/python/frozen_keras/activations.py @@ -0,0 +1,453 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Built-in activation functions.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import six + +from tensorflow.python.frozen_keras import backend as K +from tensorflow.python.frozen_keras.utils.generic_utils import deserialize_keras_object +from tensorflow.python.frozen_keras.utils.generic_utils import serialize_keras_object +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn + +# b/123041942 +# In TF 2.x, if the `tf.nn.softmax` is used as an activation function in Keras +# layers, it gets serialized as 'softmax_v2' instead of 'softmax' as the +# internal method name is returned in serialization. This results in errors in +# model exporting and loading as Keras can't find any activation function with +# the name of `softmax_v2`. + +# This dict maps the activation function name from its v2 version to its +# canonical name. +_TF_ACTIVATIONS_V2 = { + 'softmax_v2': 'softmax', +} + + +def softmax(x, axis=-1): + """Softmax converts a real vector to a vector of categorical probabilities. + + The elements of the output vector are in range (0, 1) and sum to 1. + + Each vector is handled independently. The `axis` argument sets which axis + of the input the function is applied along. + + Softmax is often used as the activation for the last + layer of a classification network because the result could be interpreted as + a probability distribution. + + The softmax of each vector x is calculated by `exp(x)/tf.reduce_sum(exp(x))`. + The input values in are the log-odds of the resulting probability. + + Arguments: + x : Input tensor. + axis: Integer, axis along which the softmax normalization is applied. + + Returns: + Tensor, output of softmax transformation (all values are non-negative + and sum to 1). + + Raises: + ValueError: In case `dim(x) == 1`. + """ + ndim = K.ndim(x) + if ndim == 2: + return nn.softmax(x) + elif ndim > 2: + e = math_ops.exp(x - math_ops.reduce_max(x, axis=axis, keepdims=True)) + s = math_ops.reduce_sum(e, axis=axis, keepdims=True) + return e / s + else: + raise ValueError('Cannot apply softmax to a tensor that is 1D. ' + 'Received input: %s' % (x,)) + + +def elu(x, alpha=1.0): + """Exponential linear unit. + + Arguments: + x: Input tensor. + alpha: A scalar, slope of negative section. + + Returns: + The exponential linear activation: `x` if `x > 0` and + `alpha * (exp(x)-1)` if `x < 0`. + + Reference: + - [Fast and Accurate Deep Network Learning by Exponential + Linear Units (ELUs)](https://arxiv.org/abs/1511.07289) + """ + return K.elu(x, alpha) + + +def selu(x): + """Scaled Exponential Linear Unit (SELU). + + The Scaled Exponential Linear Unit (SELU) activation function is: + `scale * x` if `x > 0` and `scale * alpha * (exp(x) - 1)` if `x < 0` + where `alpha` and `scale` are pre-defined constants + (`alpha = 1.67326324` + and `scale = 1.05070098`). + The SELU activation function multiplies `scale` > 1 with the + `[elu](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/activations/elu)` + (Exponential Linear Unit (ELU)) to ensure a slope larger than one + for positive net inputs. + + The values of `alpha` and `scale` are + chosen so that the mean and variance of the inputs are preserved + between two consecutive layers as long as the weights are initialized + correctly (see [`lecun_normal` initialization] + (https://www.tensorflow.org/api_docs/python/tf/keras/initializers/lecun_normal)) + and the number of inputs is "large enough" + (see references for more information). + + ![]https://cdn-images-1.medium.com/max/1600/1*m0e8lZU_Zrkh4ESfQkY2Pw.png + (Courtesy: Blog on Towards DataScience at + https://towardsdatascience.com/selu-make-fnns-great-again-snn-8d61526802a9) + + Example Usage: + + >>> n_classes = 10 #10-class problem + >>> from tensorflow.python.keras.layers import Dense + >>> model = tf.keras.Sequential() + >>> model.add(Dense(64, kernel_initializer='lecun_normal', + ... activation='selu', input_shape=(28, 28, 1))) + >>> model.add(Dense(32, kernel_initializer='lecun_normal', + ... activation='selu')) + >>> model.add(Dense(16, kernel_initializer='lecun_normal', + ... activation='selu')) + >>> model.add(Dense(n_classes, activation='softmax')) + + Arguments: + x: A tensor or variable to compute the activation function for. + + Returns: + The scaled exponential unit activation: `scale * elu(x, alpha)`. + + # Note + - To be used together with the initialization "[lecun_normal] + (https://www.tensorflow.org/api_docs/python/tf/keras/initializers/lecun_normal)". + - To be used together with the dropout variant "[AlphaDropout] + (https://www.tensorflow.org/api_docs/python/tf/keras/layers/AlphaDropout)". + + References: + [Self-Normalizing Neural Networks (Klambauer et al, 2017)] + (https://arxiv.org/abs/1706.02515) + """ + return nn.selu(x) + + +def softplus(x): + """Softplus activation function. + + Arguments: + x: Input tensor. + + Returns: + The softplus activation: `log(exp(x) + 1)`. + """ + return nn.softplus(x) + + +def softsign(x): + """Softsign activation function. + + Arguments: + x: Input tensor. + + Returns: + The softsign activation: `x / (abs(x) + 1)`. + """ + return nn.softsign(x) + + +def swish(x): + """Swish activation function. + + Arguments: + x: Input tensor. + + Returns: + The swish activation applied to `x`. + """ + return nn.swish(x) + + +def relu(x, alpha=0., max_value=None, threshold=0): + """Applies the rectified linear unit activation function. + + With default values, this returns the standard ReLU activation: + `max(x, 0)`, the element-wise maximum of 0 and the input tensor. + + Modifying default parameters allows you to use non-zero thresholds, + change the max value of the activation, + and to use a non-zero multiple of the input for values below the threshold. + + For example: + + >>> foo = tf.constant([-10, -5, 0.0, 5, 10], dtype = tf.float32) + >>> tf.keras.activations.relu(foo).numpy() + array([ 0., 0., 0., 5., 10.], dtype=float32) + >>> tf.keras.activations.relu(foo, alpha=0.5).numpy() + array([-5. , -2.5, 0. , 5. , 10. ], dtype=float32) + >>> tf.keras.activations.relu(foo, max_value=5).numpy() + array([0., 0., 0., 5., 5.], dtype=float32) + >>> tf.keras.activations.relu(foo, threshold=5).numpy() + array([-0., -0., 0., 0., 10.], dtype=float32) + + Arguments: + x: Input `tensor` or `variable`. + alpha: A `float` that governs the slope for values lower than the + threshold. + max_value: A `float` that sets the saturation threshold (the largest value + the function will return). + threshold: A `float` giving the threshold value of the activation function + below which values will be damped or set to zero. + + Returns: + A `Tensor` representing the input tensor, + transformed by the relu activation function. + Tensor will be of the same shape and dtype of input `x`. + """ + return K.relu(x, alpha=alpha, max_value=max_value, threshold=threshold) + + +def tanh(x): + """Hyperbolic tangent activation function. + + For example: + + >>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32) + >>> b = tf.keras.activations.tanh(a) + >>> b.numpy() + array([-0.9950547, -0.7615942, 0. , 0.7615942, 0.9950547], + dtype=float32) + + Arguments: + x: Input tensor. + + Returns: + Tensor of same shape and dtype of input `x`, with tanh activation: + `tanh(x) = sinh(x)/cosh(x) = ((exp(x) - exp(-x))/(exp(x) + exp(-x)))`. + """ + return nn.tanh(x) + + +def sigmoid(x): + """Sigmoid activation function. + + Applies the sigmoid activation function. The sigmoid function is defined as + 1 divided by (1 + exp(-x)). It's curve is like an "S" and is like a smoothed + version of the Heaviside (Unit Step Function) function. For small values + (<-5) the sigmoid returns a value close to zero and for larger values (>5) + the result of the function gets close to 1. + + Sigmoid is equivalent to a 2-element Softmax, where the second element is + assumed to be zero. + + For example: + + >>> a = tf.constant([-20, -1.0, 0.0, 1.0, 20], dtype = tf.float32) + >>> b = tf.keras.activations.sigmoid(a) + >>> b.numpy() >= 0.0 + array([ True, True, True, True, True]) + + Arguments: + x: Input tensor. + + Returns: + Tensor with the sigmoid activation: `(1.0 / (1.0 + exp(-x)))`. + Tensor will be of same shape and dtype of input `x`. + """ + return nn.sigmoid(x) + + +def exponential(x): + """Exponential activation function. + + For example: + + >>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32) + >>> b = tf.keras.activations.exponential(a) + >>> b.numpy() + array([ 0.04978707, 0.36787945, 1. , 2.7182817 , 20.085537 ], + dtype=float32) + + Arguments: + x: Input tensor. + + Returns: + Tensor with exponential activation: `exp(x)`. Tensor will be of same + shape and dtype of input `x`. + """ + return math_ops.exp(x) + + +def hard_sigmoid(x): + """Hard sigmoid activation function. + + Faster to compute than sigmoid activation. + + For example: + + >>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32) + >>> b = tf.keras.activations.hard_sigmoid(a) + >>> b.numpy() + array([0. , 0.3, 0.5, 0.7, 1. ], dtype=float32) + + Arguments: + x: Input tensor. + + Returns: + The hard sigmoid activation: + + - `0` if `x < -2.5` + - `1` if `x > 2.5` + - `0.2 * x + 0.5` if `-2.5 <= x <= 2.5`. + """ + return K.hard_sigmoid(x) + + +def linear(x): + """Linear activation function. + + For example: + + >>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32) + >>> b = tf.keras.activations.linear(a) + >>> b.numpy() + array([-3., -1., 0., 1., 3.], dtype=float32) + + Arguments: + x: Input tensor. + + Returns: + the input unmodified. + """ + return x + + +def serialize(activation): + """Returns name attribute (`__name__`) of function. + + Arguments: + activation : Function + + Returns: + String denoting the name attribute of the input function + + For example: + + >>> tf.keras.activations.serialize(tf.keras.activations.tanh) + 'tanh' + >>> tf.keras.activations.serialize(tf.keras.activations.sigmoid) + 'sigmoid' + >>> tf.keras.activations.serialize('abcd') + Traceback (most recent call last): + ... + ValueError: ('Cannot serialize', 'abcd') + + Raises: + ValueError: The input function is not a valid one. + """ + if (hasattr(activation, '__name__') and + activation.__name__ in _TF_ACTIVATIONS_V2): + return _TF_ACTIVATIONS_V2[activation.__name__] + return serialize_keras_object(activation) + + +def deserialize(name, custom_objects=None): + """Returns activation function denoted by input string. + + Arguments: + x : String + + Returns: + TensorFlow Activation function denoted by input string. + + For example: + + >>> tf.keras.activations.deserialize('linear') + + >>> tf.keras.activations.deserialize('sigmoid') + + >>> tf.keras.activations.deserialize('abcd') + Traceback (most recent call last): + ... + ValueError: Unknown activation function:abcd + + Args: + name: The name of the activation function. + custom_objects: A {name:value} dictionary for activations not build into + keras. + + Raises: + ValueError: `Unknown activation function` if the input string does not + denote any defined Tensorflow activation function. + """ + return deserialize_keras_object( + name, + module_objects=globals(), + custom_objects=custom_objects, + printable_module_name='activation function') + + +def get(identifier): + """Returns function. + + Arguments: + identifier: Function or string + + Returns: + Activation function denoted by input: + - `Linear activation function` if input is `None`. + - Function corresponding to the input string or input function. + + For example: + + >>> tf.keras.activations.get('softmax') + + >>> tf.keras.activations.get(tf.keras.activations.softmax) + + >>> tf.keras.activations.get(None) + + >>> tf.keras.activations.get(abs) + + >>> tf.keras.activations.get('abcd') + Traceback (most recent call last): + ... + ValueError: Unknown activation function:abcd + + Raises: + ValueError: Input is an unknown function or string, i.e., the input does + not denote any defined function. + """ + if identifier is None: + return linear + if isinstance(identifier, six.string_types): + identifier = str(identifier) + return deserialize(identifier) + elif callable(identifier): + return identifier + elif isinstance(identifier, dict): + return deserialize_keras_object( + identifier, printable_module_name='activation') + else: + raise TypeError( + 'Could not interpret activation function identifier: {}'.format( + repr(identifier)))